[1488057.711176] CPU: 63 PID: 54246 Comm: ptlrpcd_07_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488057.711177] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488057.711178] task: ffff8abafc522080 ti: ffff8abaf4f54000 task.ti: ffff8abaf4f54000 [1488057.711182] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488057.711183] RSP: 0018:ffff8abaf4f57b78 EFLAGS: 00000246 [1488057.711183] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001f90000 [1488057.711184] RDX: ffff8a9affa9b780 RSI: 0000000000910000 RDI: ffff8adafa7b1b00 [1488057.711185] RBP: ffff8abaf4f57b78 R08: ffff8afb3f9db780 R09: 0000000000000000 [1488057.711185] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aed569e2a00 [1488057.711186] R13: 0005c4aa17c51860 R14: ffff8aed24f64b00 R15: 0000000000000007 [1488057.711187] FS: 00007ea1d389b700(0000) GS:ffff8afb3f9c0000(0000) knlGS:0000000000000000 [1488057.711188] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488057.711189] CR2: 00007ecf3e8c7ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488057.711190] Call Trace: [1488057.711192] [] queued_spin_lock_slowpath+0xb/0xf [1488057.711193] [] _raw_spin_lock+0x20/0x30 [1488057.711201] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488057.711211] [] LNetMDUnlink+0x78/0x180 [lnet] [1488057.711250] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488057.711287] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488057.711324] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488057.711362] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488057.711364] [] ? wake_up_state+0x20/0x20 [1488057.711402] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488057.711404] [] kthread+0xd1/0xe0 [1488057.711406] [] ? insert_kthread_work+0x40/0x40 [1488057.711408] [] ret_from_fork_nospec_begin+0xe/0x21 [1488057.711410] [] ? insert_kthread_work+0x40/0x40 [1488057.711430] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488057.758142] NMI watchdog: BUG: soft lockup - CPU#68 stuck for 22s! [ptlrpcd_02_02:54182] [1488057.758168] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488057.758175] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488057.758178] CPU: 68 PID: 54182 Comm: ptlrpcd_02_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488057.758178] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488057.758180] task: ffff8abaee641040 ti: ffff8abafcdf0000 task.ti: ffff8abafcdf0000 [1488057.758183] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488057.758184] RSP: 0018:ffff8abafcdf3b78 EFLAGS: 00000246 [1488057.758185] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002210000 [1488057.758186] RDX: ffff8aaaffbdb780 RSI: 0000000001e10001 RDI: ffff8adafa7b1b00 [1488057.758187] RBP: ffff8abafcdf3b78 R08: ffff8aaaffc1b780 R09: 0000000000000000 [1488057.758187] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa4ef142800 [1488057.758188] R13: 0005c4aa17cce1a0 R14: ffff8aa4721f3900 R15: 0000000000000007 [1488057.758190] FS: 00007ea149112700(0000) GS:ffff8aaaffc00000(0000) knlGS:0000000000000000 [1488057.758191] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488057.758191] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488057.758192] Call Trace: [1488057.758195] [] queued_spin_lock_slowpath+0xb/0xf [1488057.758197] [] _raw_spin_lock+0x20/0x30 [1488057.758204] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488057.758215] [] LNetMDUnlink+0x78/0x180 [lnet] [1488057.758253] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488057.758291] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488057.758328] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488057.758366] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488057.758368] [] ? wake_up_state+0x20/0x20 [1488057.758407] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488057.758408] [] kthread+0xd1/0xe0 [1488057.758410] [] ? insert_kthread_work+0x40/0x40 [1488057.758412] [] ret_from_fork_nospec_begin+0xe/0x21 [1488057.758414] [] ? insert_kthread_work+0x40/0x40 [1488057.758433] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488057.901153] NMI watchdog: BUG: soft lockup - CPU#84 stuck for 22s! [ptlrpcd_02_00:54180] [1488057.901178] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488057.901185] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488057.901187] CPU: 84 PID: 54180 Comm: ptlrpcd_02_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488057.901188] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488057.901189] task: ffff8abaf4dae180 ti: ffff8abaecadc000 task.ti: ffff8abaecadc000 [1488057.901192] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488057.901193] RSP: 0018:ffff8abaecadfb78 EFLAGS: 00000246 [1488057.901194] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002a10000 [1488057.901195] RDX: ffff8aaaffb1b780 RSI: 0000000001210001 RDI: ffff8adafa7b1b00 [1488057.901195] RBP: ffff8abaecadfb78 R08: ffff8aaaffc9b780 R09: 0000000000000000 [1488057.901196] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9bffcd7200 [1488057.901197] R13: 0005c4aa17cd8670 R14: ffff8aa639e9aa00 R15: 0000000000000007 [1488057.901199] FS: 00007ea1c8885700(0000) GS:ffff8aaaffc80000(0000) knlGS:0000000000000000 [1488057.901200] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488057.901201] CR2: 00007ee6f34aa006 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488057.901201] Call Trace: [1488057.901204] [] queued_spin_lock_slowpath+0xb/0xf [1488057.901206] [] _raw_spin_lock+0x20/0x30 [1488057.901213] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488057.901224] [] LNetMDUnlink+0x78/0x180 [lnet] [1488057.901261] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488057.901298] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488057.901335] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488057.901373] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488057.901375] [] ? wake_up_state+0x20/0x20 [1488057.901413] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488057.901415] [] kthread+0xd1/0xe0 [1488057.901417] [] ? insert_kthread_work+0x40/0x40 [1488057.901419] [] ret_from_fork_nospec_begin+0xe/0x21 [1488057.901420] [] ? insert_kthread_work+0x40/0x40 [1488057.901440] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488057.993155] NMI watchdog: BUG: soft lockup - CPU#94 stuck for 23s! [ptlrpcd_03_02:54194] [1488057.993180] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488057.993187] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488057.993189] CPU: 94 PID: 54194 Comm: ptlrpcd_03_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488057.993189] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488057.993190] task: ffff8abaebafe180 ti: ffff8abaf8e10000 task.ti: ffff8abaf8e10000 [1488057.993193] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488057.993194] RSP: 0018:ffff8abaf8e13b78 EFLAGS: 00000246 [1488057.993195] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002f10000 [1488057.993196] RDX: ffff8a8aff2db780 RSI: 0000000000c10001 RDI: ffff8adafa7b1b00 [1488057.993196] RBP: ffff8abaf8e13b78 R08: ffff8abaffcdb780 R09: 0000000000000000 [1488057.993197] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab1562e0600 [1488057.993198] R13: 0005c4aa17cca680 R14: ffff8ab8fa384b00 R15: 0000000000000007 [1488057.993199] FS: 00007ea1d1897700(0000) GS:ffff8abaffcc0000(0000) knlGS:0000000000000000 [1488057.993200] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488057.993200] CR2: 00007f47fafb9014 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488057.993201] Call Trace: [1488057.993203] [] queued_spin_lock_slowpath+0xb/0xf [1488057.993205] [] _raw_spin_lock+0x20/0x30 [1488057.993212] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488057.993222] [] LNetMDUnlink+0x78/0x180 [lnet] [1488057.993260] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488057.993297] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488057.993334] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488057.993372] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488057.993374] [] ? wake_up_state+0x20/0x20 [1488057.993412] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488057.993413] [] kthread+0xd1/0xe0 [1488057.993415] [] ? insert_kthread_work+0x40/0x40 [1488057.993416] [] ret_from_fork_nospec_begin+0xe/0x21 [1488057.993418] [] ? insert_kthread_work+0x40/0x40 [1488057.993437] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.155369] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ptlrpcd_01_10:54178] [1488061.155395] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.155402] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.155405] CPU: 2 PID: 54178 Comm: ptlrpcd_01_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.155406] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.155407] task: ffff8abaf4dac100 ti: ffff8abafd6fc000 task.ti: ffff8abafd6fc000 [1488061.155410] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.155411] RSP: 0018:ffff8abafd6ffb78 EFLAGS: 00000246 [1488061.155412] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000110000 [1488061.155413] RDX: ffff8a9affc1b780 RSI: 0000000002110001 RDI: ffff8adafa7b1b00 [1488061.155414] RBP: ffff8abafd6ffb78 R08: ffff8a9affa1b780 R09: 0000000000000000 [1488061.155414] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a94eddae200 [1488061.155415] R13: 0005c4aa179f63f0 R14: ffff8a97eef74200 R15: 0000000000000007 [1488061.155417] FS: 00007ea1cb08a700(0000) GS:ffff8a9affa00000(0000) knlGS:0000000000000000 [1488061.155418] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.155418] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.155419] Call Trace: [1488061.155421] [] queued_spin_lock_slowpath+0xb/0xf [1488061.155423] [] _raw_spin_lock+0x20/0x30 [1488061.155431] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.155442] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.155479] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.155517] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.155553] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.155592] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.155594] [] ? wake_up_state+0x20/0x20 [1488061.155632] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.155634] [] kthread+0xd1/0xe0 [1488061.155636] [] ? insert_kthread_work+0x40/0x40 [1488061.155637] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.155639] [] ? insert_kthread_work+0x40/0x40 [1488061.155659] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.174368] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ptlrpcd_02_10:54190] [1488061.174393] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.174400] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.174403] CPU: 4 PID: 54190 Comm: ptlrpcd_02_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.174403] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.174404] task: ffff8abaebafa080 ti: ffff8abaeca8c000 task.ti: ffff8abaeca8c000 [1488061.174407] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.174408] RSP: 0018:ffff8abaeca8fb78 EFLAGS: 00000246 [1488061.174409] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000210000 [1488061.174410] RDX: ffff8adaffc9b780 RSI: 0000000002990001 RDI: ffff8adafa7b1b00 [1488061.174411] RBP: ffff8abaeca8fb78 R08: ffff8aaaffa1b780 R09: 0000000000000000 [1488061.174412] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8fde494e00 [1488061.174413] R13: 0005c4aa17cd4900 R14: ffff8aa3ff308000 R15: 0000000000000007 [1488061.174414] FS: 00007ea1d509e700(0000) GS:ffff8aaaffa00000(0000) knlGS:0000000000000000 [1488061.174415] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.174416] CR2: 0000000000453b20 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.174417] Call Trace: [1488061.174419] [] queued_spin_lock_slowpath+0xb/0xf [1488061.174421] [] _raw_spin_lock+0x20/0x30 [1488061.174429] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.174439] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.174477] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.174516] [] ? deuuidify+0x90/0x90 [ptlrpc] [1488061.174553] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.174590] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.174628] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.174630] [] ? wake_up_state+0x20/0x20 [1488061.174668] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.174670] [] kthread+0xd1/0xe0 [1488061.174671] [] ? insert_kthread_work+0x40/0x40 [1488061.174673] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.174675] [] ? insert_kthread_work+0x40/0x40 [1488061.174695] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.182372] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ptlrpcd_06_01:54229] [1488061.182400] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.182408] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.182411] CPU: 5 PID: 54229 Comm: ptlrpcd_06_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.182412] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.182413] task: ffff8abafb33e180 ti: ffff8abaf5350000 task.ti: ffff8abaf5350000 [1488061.182417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.182418] RSP: 0018:ffff8abaf5353b78 EFLAGS: 00000246 [1488061.182419] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000290000 [1488061.182420] RDX: ffff8aeaffcdb780 RSI: 0000000002e90001 RDI: ffff8adafa7b1b00 [1488061.182421] RBP: ffff8abaf5353b78 R08: ffff8aeaffa1b780 R09: 0000000000000000 [1488061.182422] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aea9ae2e200 [1488061.182423] R13: 0005c4aa17d2e8f0 R14: ffff8ae450b52100 R15: 0000000000000007 [1488061.182425] FS: 00007ea1bb06a700(0000) GS:ffff8aeaffa00000(0000) knlGS:0000000000000000 [1488061.182426] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.182427] CR2: 00007f6e7e5bf590 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.182428] Call Trace: [1488061.182430] [] queued_spin_lock_slowpath+0xb/0xf [1488061.182432] [] _raw_spin_lock+0x20/0x30 [1488061.182440] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.182451] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.182489] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.182527] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.182564] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.182603] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.182606] [] ? wake_up_state+0x20/0x20 [1488061.182644] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.182646] [] kthread+0xd1/0xe0 [1488061.182648] [] ? insert_kthread_work+0x40/0x40 [1488061.182650] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.182652] [] ? insert_kthread_work+0x40/0x40 [1488061.182672] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.191371] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ptlrpcd_03_11:54203] [1488061.191399] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.191406] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.191409] CPU: 6 PID: 54203 Comm: ptlrpcd_03_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.191410] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.191410] task: ffff8aaba9651040 ti: ffff8abaed288000 task.ti: ffff8abaed288000 [1488061.191414] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.191415] RSP: 0018:ffff8abaed28bb78 EFLAGS: 00000246 [1488061.191416] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000310000 [1488061.191417] RDX: ffff8a9affadb780 RSI: 0000000000d10001 RDI: ffff8adafa7b1b00 [1488061.191418] RBP: ffff8abaed28bb78 R08: ffff8abaffa1b780 R09: 0000000000000000 [1488061.191418] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab36218f200 [1488061.191419] R13: 0005c4aa17d36130 R14: ffff8ab5a74fbc00 R15: 0000000000000007 [1488061.191421] FS: 00007ea1dc8ad700(0000) GS:ffff8abaffa00000(0000) knlGS:0000000000000000 [1488061.191421] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.191422] CR2: 00007f5ff48d9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.191423] Call Trace: [1488061.191425] [] queued_spin_lock_slowpath+0xb/0xf [1488061.191427] [] _raw_spin_lock+0x20/0x30 [1488061.191435] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.191446] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.191483] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.191521] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.191557] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.191596] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.191598] [] ? wake_up_state+0x20/0x20 [1488061.191636] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.191638] [] kthread+0xd1/0xe0 [1488061.191640] [] ? insert_kthread_work+0x40/0x40 [1488061.191642] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.191643] [] ? insert_kthread_work+0x40/0x40 [1488061.191663] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.201368] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ptlrpcd_07_03:54243] [1488061.201398] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.201405] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.201408] CPU: 7 PID: 54243 Comm: ptlrpcd_07_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.201409] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.201410] task: ffff8abafbc7e180 ti: ffff8abafbb24000 task.ti: ffff8abafbb24000 [1488061.201414] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.201415] RSP: 0018:ffff8abafbb27b78 EFLAGS: 00000246 [1488061.201416] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000390000 [1488061.201417] RDX: ffff8acaffcdb780 RSI: 0000000002c90001 RDI: ffff8adafa7b1b00 [1488061.201418] RBP: ffff8abafbb27b78 R08: ffff8afb3f81b780 R09: 0000000000000000 [1488061.201419] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af1a1f2a000 [1488061.201420] R13: 0005c4aa17ccfa80 R14: ffff8af090a4da00 R15: 0000000000000007 [1488061.201422] FS: 00007ea1ca889700(0000) GS:ffff8afb3f800000(0000) knlGS:0000000000000000 [1488061.201422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.201424] CR2: 00007ee55cb4bff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.201425] Call Trace: [1488061.201427] [] queued_spin_lock_slowpath+0xb/0xf [1488061.201429] [] _raw_spin_lock+0x20/0x30 [1488061.201437] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.201448] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.201487] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.201524] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.201562] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.201601] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.201603] [] ? wake_up_state+0x20/0x20 [1488061.201642] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.201643] [] kthread+0xd1/0xe0 [1488061.201645] [] ? insert_kthread_work+0x40/0x40 [1488061.201647] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.201649] [] ? insert_kthread_work+0x40/0x40 [1488061.201669] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.209371] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ptlrpcd_00_02:54158] [1488061.209398] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.209405] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.209408] CPU: 8 PID: 54158 Comm: ptlrpcd_00_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.209409] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.209410] task: ffff8afaf5f330c0 ti: ffff8abaf6878000 task.ti: ffff8abaf6878000 [1488061.209413] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.209414] RSP: 0018:ffff8abaf687bb78 EFLAGS: 00000246 [1488061.209415] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000410000 [1488061.209416] RDX: ffff8aeaffb1b780 RSI: 0000000001290001 RDI: ffff8adafa7b1b00 [1488061.209417] RBP: ffff8abaf687bb78 R08: ffff8a8aff25b780 R09: 0000000000000000 [1488061.209418] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8bc6590200 [1488061.209419] R13: 0005c4aa187a08a0 R14: ffff8a9a09e6da00 R15: 0000000000000007 [1488061.209420] FS: 00007ea1c9086700(0000) GS:ffff8a8aff240000(0000) knlGS:0000000000000000 [1488061.209421] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.209422] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.209423] Call Trace: [1488061.209425] [] queued_spin_lock_slowpath+0xb/0xf [1488061.209427] [] _raw_spin_lock+0x20/0x30 [1488061.209435] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.209446] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.209484] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.209521] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.209558] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.209598] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.209600] [] ? wake_up_state+0x20/0x20 [1488061.209638] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.209640] [] kthread+0xd1/0xe0 [1488061.209641] [] ? insert_kthread_work+0x40/0x40 [1488061.209643] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.209645] [] ? insert_kthread_work+0x40/0x40 [1488061.209665] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.219374] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ptlrpcd_04_07:54211] [1488061.219401] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.219409] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.219411] CPU: 9 PID: 54211 Comm: ptlrpcd_04_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.219412] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.219413] task: ffff8abaf82ca080 ti: ffff8abafd344000 task.ti: ffff8abafd344000 [1488061.219417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.219417] RSP: 0018:ffff8abafd347b78 EFLAGS: 00000246 [1488061.219418] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000490000 [1488061.219419] RDX: ffff8acaffc5b780 RSI: 0000000002490001 RDI: ffff8adafa7b1b00 [1488061.219421] RBP: ffff8abafd347b78 R08: ffff8acaffa5b780 R09: 0000000000000000 [1488061.219421] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abc1b613c00 [1488061.219422] R13: 0005c4aa17cc1560 R14: ffff8ac421352a00 R15: 0000000000000007 [1488061.219424] FS: 00007ea1b0054700(0000) GS:ffff8acaffa40000(0000) knlGS:0000000000000000 [1488061.219425] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.219426] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.219427] Call Trace: [1488061.219429] [] queued_spin_lock_slowpath+0xb/0xf [1488061.219431] [] _raw_spin_lock+0x20/0x30 [1488061.219440] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.219450] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.219489] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.219527] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.219564] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.219603] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.219605] [] ? wake_up_state+0x20/0x20 [1488061.219643] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.219645] [] kthread+0xd1/0xe0 [1488061.219647] [] ? insert_kthread_work+0x40/0x40 [1488061.219649] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.219651] [] ? insert_kthread_work+0x40/0x40 [1488061.219671] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.229371] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [ptlrpcd_01_05:54173] [1488061.229399] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.229406] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.229409] CPU: 10 PID: 54173 Comm: ptlrpcd_01_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.229410] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.229411] task: ffff8abaf6346180 ti: ffff8abaf3da8000 task.ti: ffff8abaf3da8000 [1488061.229414] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.229415] RSP: 0018:ffff8abaf3dabb78 EFLAGS: 00000246 [1488061.229416] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000510000 [1488061.229417] RDX: ffff8afb3fa1b780 RSI: 0000000002390001 RDI: ffff8adafa7b1b00 [1488061.229417] RBP: ffff8abaf3dabb78 R08: ffff8a9affa5b780 R09: 0000000000000000 [1488061.229418] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a90a36b8a00 [1488061.229419] R13: 0005c4aa17a6f150 R14: ffff8a90848c4200 R15: 0000000000000007 [1488061.229421] FS: 00007ea1b2859700(0000) GS:ffff8a9affa40000(0000) knlGS:0000000000000000 [1488061.229422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.229423] CR2: 00007ee89ec1e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.229423] Call Trace: [1488061.229426] [] queued_spin_lock_slowpath+0xb/0xf [1488061.229428] [] _raw_spin_lock+0x20/0x30 [1488061.229435] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.229446] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.229484] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.229522] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.229559] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.229598] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.229600] [] ? wake_up_state+0x20/0x20 [1488061.229639] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.229640] [] kthread+0xd1/0xe0 [1488061.229642] [] ? insert_kthread_work+0x40/0x40 [1488061.229644] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.229646] [] ? insert_kthread_work+0x40/0x40 [1488061.229666] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.246370] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ptlrpcd_02_03:54183] [1488061.246396] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.246403] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.246406] CPU: 12 PID: 54183 Comm: ptlrpcd_02_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.246406] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.246407] task: ffff8abaee642080 ti: ffff8abaf4528000 task.ti: ffff8abaf4528000 [1488061.246410] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.246411] RSP: 0018:ffff8abaf452bb78 EFLAGS: 00000246 [1488061.246412] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000610000 [1488061.246413] RDX: ffff8adaffb5b780 RSI: 0000000001590001 RDI: ffff8adafa7b1b00 [1488061.246414] RBP: ffff8abaf452bb78 R08: ffff8aaaffa5b780 R09: 0000000000000000 [1488061.246415] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9f7c35a000 [1488061.246416] R13: 0005c4aa17cf54e0 R14: ffff8aa6838d2a00 R15: 0000000000000007 [1488061.246417] FS: 00007fc0d2114880(0000) GS:ffff8aaaffa40000(0000) knlGS:0000000000000000 [1488061.246418] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.246419] CR2: 00007fc0cc4ae000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.246420] Call Trace: [1488061.246422] [] queued_spin_lock_slowpath+0xb/0xf [1488061.246424] [] _raw_spin_lock+0x20/0x30 [1488061.246432] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.246442] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.246480] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.246518] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.246555] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.246594] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.246596] [] ? wake_up_state+0x20/0x20 [1488061.246634] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.246636] [] kthread+0xd1/0xe0 [1488061.246637] [] ? insert_kthread_work+0x40/0x40 [1488061.246639] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.246641] [] ? insert_kthread_work+0x40/0x40 [1488061.246661] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.256375] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ptlrpcd_06_02:54230] [1488061.256403] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.256410] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.256413] CPU: 13 PID: 54230 Comm: ptlrpcd_06_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.256414] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.256415] task: ffff8abaf4780000 ti: ffff8abaf3c14000 task.ti: ffff8abaf3c14000 [1488061.256418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.256419] RSP: 0018:ffff8abaf3c17b78 EFLAGS: 00000246 [1488061.256420] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000690000 [1488061.256421] RDX: ffff8afb3fadb780 RSI: 0000000002f90001 RDI: ffff8adafa7b1b00 [1488061.256422] RBP: ffff8abaf3c17b78 R08: ffff8aeaffa5b780 R09: 0000000000000000 [1488061.256423] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aea78775000 [1488061.256424] R13: 0005c4aa17ca1d80 R14: ffff8ae05a09b300 R15: 0000000000000007 [1488061.256426] FS: 00007ea1d0094700(0000) GS:ffff8aeaffa40000(0000) knlGS:0000000000000000 [1488061.256427] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.256428] CR2: 00007eeb9290a018 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.256429] Call Trace: [1488061.256431] [] queued_spin_lock_slowpath+0xb/0xf [1488061.256433] [] _raw_spin_lock+0x20/0x30 [1488061.256441] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.256451] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.256490] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.256528] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.256565] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.256603] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.256605] [] ? wake_up_state+0x20/0x20 [1488061.256644] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.256646] [] kthread+0xd1/0xe0 [1488061.256648] [] ? insert_kthread_work+0x40/0x40 [1488061.256649] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.256652] [] ? insert_kthread_work+0x40/0x40 [1488061.256672] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.265373] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [ptlrpcd_03_00:54192] [1488061.265400] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.265407] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.265409] CPU: 14 PID: 54192 Comm: ptlrpcd_03_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.265410] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.265410] task: ffff8abaebafc100 ti: ffff8abaecb70000 task.ti: ffff8abaecb70000 [1488061.265413] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.265414] RSP: 0018:ffff8abaecb73b78 EFLAGS: 00000246 [1488061.265415] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000710000 [1488061.265415] RDX: ffff8a9affc9b780 RSI: 0000000002910001 RDI: ffff8adafa7b1b00 [1488061.265416] RBP: ffff8abaecb73b78 R08: ffff8abaffa5b780 R09: 0000000000000000 [1488061.265417] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab181efec00 [1488061.265418] R13: 0005c4aa17cb8650 R14: ffff8ab98a532700 R15: 0000000000000007 [1488061.265419] FS: 00007ea1d309a700(0000) GS:ffff8abaffa40000(0000) knlGS:0000000000000000 [1488061.265420] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.265420] CR2: 00007ef9d5eb2334 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.265421] Call Trace: [1488061.265423] [] queued_spin_lock_slowpath+0xb/0xf [1488061.265425] [] _raw_spin_lock+0x20/0x30 [1488061.265432] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.265442] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.265480] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.265517] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.265554] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.265592] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.265594] [] ? wake_up_state+0x20/0x20 [1488061.265632] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.265633] [] kthread+0xd1/0xe0 [1488061.265635] [] ? insert_kthread_work+0x40/0x40 [1488061.265636] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.265638] [] ? insert_kthread_work+0x40/0x40 [1488061.265657] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.273376] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ptlrpcd_07_10:54250] [1488061.273404] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.273412] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.273414] CPU: 15 PID: 54250 Comm: ptlrpcd_07_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.273415] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.273416] task: ffff8abafc526180 ti: ffff8abafbe28000 task.ti: ffff8abafbe28000 [1488061.273419] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.273421] RSP: 0018:ffff8abafbe2bb78 EFLAGS: 00000246 [1488061.273422] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000790000 [1488061.273422] RDX: ffff8aaaffc5b780 RSI: 0000000002610001 RDI: ffff8adafa7b1b00 [1488061.273423] RBP: ffff8abafbe2bb78 R08: ffff8afb3f85b780 R09: 0000000000000000 [1488061.273425] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af5d3316800 [1488061.273426] R13: 0005c4aa17c32eb0 R14: ffff8af0e4b25400 R15: 0000000000000007 [1488061.273427] FS: 00007ea1b9066700(0000) GS:ffff8afb3f840000(0000) knlGS:0000000000000000 [1488061.273428] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.273429] CR2: 00007ecf3e917ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.273430] Call Trace: [1488061.273432] [] queued_spin_lock_slowpath+0xb/0xf [1488061.273434] [] _raw_spin_lock+0x20/0x30 [1488061.273442] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.273453] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.273491] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.273529] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.273566] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.273605] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.273607] [] ? wake_up_state+0x20/0x20 [1488061.273645] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.273647] [] kthread+0xd1/0xe0 [1488061.273649] [] ? insert_kthread_work+0x40/0x40 [1488061.273651] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.273653] [] ? insert_kthread_work+0x40/0x40 [1488061.273673] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.282375] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [ptlrpcd_00_07:54163] [1488061.282402] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.282409] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.282412] CPU: 16 PID: 54163 Comm: ptlrpcd_00_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.282413] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.282414] task: ffff8abaf815b0c0 ti: ffff8abaf8e1c000 task.ti: ffff8abaf8e1c000 [1488061.282417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.282418] RSP: 0018:ffff8abaf8e1fb78 EFLAGS: 00000246 [1488061.282419] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000810000 [1488061.282420] RDX: ffff8adaffadb780 RSI: 0000000000d90001 RDI: ffff8adafa7b1b00 [1488061.282421] RBP: ffff8abaf8e1fb78 R08: ffff8a8aff29b780 R09: 0000000000000000 [1488061.282422] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8bc6592200 [1488061.282423] R13: 0005c4aa187a0d10 R14: ffff8a9a09e6bf00 R15: 0000000000000007 [1488061.282424] FS: 00007ea1d309a700(0000) GS:ffff8a8aff280000(0000) knlGS:0000000000000000 [1488061.282425] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.282426] CR2: 00007f47239c9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.282427] Call Trace: [1488061.282429] [] queued_spin_lock_slowpath+0xb/0xf [1488061.282431] [] _raw_spin_lock+0x20/0x30 [1488061.282439] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.282449] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.282487] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.282524] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.282561] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.282599] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.282601] [] ? wake_up_state+0x20/0x20 [1488061.282639] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.282641] [] kthread+0xd1/0xe0 [1488061.282643] [] ? insert_kthread_work+0x40/0x40 [1488061.282645] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.282647] [] ? insert_kthread_work+0x40/0x40 [1488061.282667] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.291374] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ptlrpcd_04_05:54209] [1488061.291402] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.291409] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.291412] CPU: 17 PID: 54209 Comm: ptlrpcd_04_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.291413] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.291415] task: ffff8abaf82c8000 ti: ffff8abafcde8000 task.ti: ffff8abafcde8000 [1488061.291419] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488061.291420] RSP: 0018:ffff8abafcdebb78 EFLAGS: 00000246 [1488061.291420] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000890000 [1488061.291422] RDX: ffff8aeaffb9b780 RSI: 0000000001a90001 RDI: ffff8adafa7b1b00 [1488061.291423] RBP: ffff8abafcdebb78 R08: ffff8acaffa9b780 R09: 0000000000000000 [1488061.291423] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac1a1ad1800 [1488061.291425] R13: 0005c4aa17cc9500 R14: ffff8ac1a1b0b300 R15: 0000000000000007 [1488061.291426] FS: 00007ea1be871700(0000) GS:ffff8acaffa80000(0000) knlGS:0000000000000000 [1488061.291427] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.291428] CR2: 00007ef98c7b802a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.291429] Call Trace: [1488061.291431] [] queued_spin_lock_slowpath+0xb/0xf [1488061.291434] [] _raw_spin_lock+0x20/0x30 [1488061.291441] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.291452] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.291491] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.291529] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.291566] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.291605] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.291607] [] ? wake_up_state+0x20/0x20 [1488061.291646] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.291648] [] kthread+0xd1/0xe0 [1488061.291650] [] ? insert_kthread_work+0x40/0x40 [1488061.291651] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.291654] [] ? insert_kthread_work+0x40/0x40 [1488061.291674] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488061.301380] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [ptlrpcd_01_09:54177] [1488061.301407] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.301415] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.301417] CPU: 18 PID: 54177 Comm: ptlrpcd_01_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.301418] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.301419] task: ffff8abaf4dab0c0 ti: ffff8abaf9440000 task.ti: ffff8abaf9440000 [1488061.301422] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.301423] RSP: 0018:ffff8abaf9443b78 EFLAGS: 00000246 [1488061.301424] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000910000 [1488061.301425] RDX: ffff8acaffbdb780 RSI: 0000000001c90001 RDI: ffff8adafa7b1b00 [1488061.301426] RBP: ffff8abaf9443b78 R08: ffff8a9affa9b780 R09: 0000000000000000 [1488061.301427] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a931aadea00 [1488061.301427] R13: 0005c4aa17a58110 R14: ffff8a8c35a01800 R15: 0000000000000007 [1488061.301429] FS: 00007ea1d409c700(0000) GS:ffff8a9affa80000(0000) knlGS:0000000000000000 [1488061.301430] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.301431] CR2: 00007fbad3d81000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.301431] Call Trace: [1488061.301433] [] queued_spin_lock_slowpath+0xb/0xf [1488061.301436] [] _raw_spin_lock+0x20/0x30 [1488061.301443] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.301454] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.301492] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.301529] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.301566] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.301604] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.301606] [] ? wake_up_state+0x20/0x20 [1488061.301644] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.301646] [] kthread+0xd1/0xe0 [1488061.301648] [] ? insert_kthread_work+0x40/0x40 [1488061.301650] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.301651] [] ? insert_kthread_work+0x40/0x40 [1488061.301672] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.318377] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ptlrpcd_02_07:54187] [1488061.318402] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.318410] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.318412] CPU: 20 PID: 54187 Comm: ptlrpcd_02_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.318413] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.318414] task: ffff8abaee646180 ti: ffff8abaf9b9c000 task.ti: ffff8abaf9b9c000 [1488061.318417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.318417] RSP: 0018:ffff8abaf9b9fb78 EFLAGS: 00000246 [1488061.318418] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000a10000 [1488061.318419] RDX: ffff8a8aff39b780 RSI: 0000000001810001 RDI: ffff8adafa7b1b00 [1488061.318420] RBP: ffff8abaf9b9fb78 R08: ffff8aaaffa9b780 R09: 0000000000000000 [1488061.318421] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa82d59ee00 [1488061.318422] R13: 0005c4aa17cfe700 R14: ffff8a9b7a153f00 R15: 0000000000000007 [1488061.318423] FS: 00007ea1c8885700(0000) GS:ffff8aaaffa80000(0000) knlGS:0000000000000000 [1488061.318424] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.318425] CR2: 00007ee8a005c02a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.318426] Call Trace: [1488061.318428] [] queued_spin_lock_slowpath+0xb/0xf [1488061.318429] [] _raw_spin_lock+0x20/0x30 [1488061.318437] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.318448] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.318486] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.318488] [] ? del_timer_sync+0x52/0x60 [1488061.318525] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.318562] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.318601] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.318603] [] ? wake_up_state+0x20/0x20 [1488061.318641] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.318642] [] kthread+0xd1/0xe0 [1488061.318644] [] ? insert_kthread_work+0x40/0x40 [1488061.318646] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.318648] [] ? insert_kthread_work+0x40/0x40 [1488061.318668] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.328375] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ptlrpcd_06_11:54239] [1488061.328403] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.328410] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.328413] CPU: 21 PID: 54239 Comm: ptlrpcd_06_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.328414] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.328415] task: ffff8abafbc7a080 ti: ffff8abaebaa4000 task.ti: ffff8abaebaa4000 [1488061.328418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.328419] RSP: 0018:ffff8abaebaa7b78 EFLAGS: 00000246 [1488061.328420] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000a90000 [1488061.328421] RDX: ffff8acaffc1b780 RSI: 0000000002090001 RDI: ffff8adafa7b1b00 [1488061.328422] RBP: ffff8abaebaa7b78 R08: ffff8aeaffa9b780 R09: 0000000000000000 [1488061.328423] R10: 0000000000000000 R11: 000000000000000f R12: ffff8add0f6dfe00 [1488061.328424] R13: 0005c4aa17cdc9a0 R14: ffff8ae085aa2700 R15: 0000000000000007 [1488061.328426] FS: 00007ea1b7863700(0000) GS:ffff8aeaffa80000(0000) knlGS:0000000000000000 [1488061.328427] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.328428] CR2: 00007eab533f5000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.328429] Call Trace: [1488061.328431] [] queued_spin_lock_slowpath+0xb/0xf [1488061.328433] [] _raw_spin_lock+0x20/0x30 [1488061.328441] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.328452] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.328490] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.328528] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.328565] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.328604] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.328606] [] ? wake_up_state+0x20/0x20 [1488061.328645] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.328647] [] kthread+0xd1/0xe0 [1488061.328649] [] ? insert_kthread_work+0x40/0x40 [1488061.328650] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.328653] [] ? insert_kthread_work+0x40/0x40 [1488061.328675] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.338382] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [ptlrpcd_03_01:54193] [1488061.338410] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.338418] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.338420] CPU: 22 PID: 54193 Comm: ptlrpcd_03_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.338420] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.338422] task: ffff8abaebafd140 ti: ffff8abafd5a4000 task.ti: ffff8abafd5a4000 [1488061.338425] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.338426] RSP: 0018:ffff8abafd5a7b78 EFLAGS: 00000246 [1488061.338427] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000b10000 [1488061.338427] RDX: ffff8adaffc1b780 RSI: 0000000002190001 RDI: ffff8adafa7b1b00 [1488061.338428] RBP: ffff8abafd5a7b78 R08: ffff8abaffa9b780 R09: 0000000000000000 [1488061.338429] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aba39766000 [1488061.338430] R13: 0005c4aa17d34c00 R14: ffff8ab89e37b000 R15: 0000000000000007 [1488061.338431] FS: 00007ea1c8885700(0000) GS:ffff8abaffa80000(0000) knlGS:0000000000000000 [1488061.338432] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.338433] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.338434] Call Trace: [1488061.338436] [] queued_spin_lock_slowpath+0xb/0xf [1488061.338438] [] _raw_spin_lock+0x20/0x30 [1488061.338446] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.338457] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.338495] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.338532] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.338570] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.338608] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.338610] [] ? wake_up_state+0x20/0x20 [1488061.338649] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.338651] [] kthread+0xd1/0xe0 [1488061.338653] [] ? insert_kthread_work+0x40/0x40 [1488061.338655] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.338656] [] ? insert_kthread_work+0x40/0x40 [1488061.338676] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.346381] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ptlrpcd_07_05:54245] [1488061.346408] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.346416] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.346419] CPU: 23 PID: 54245 Comm: ptlrpcd_07_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.346420] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.346421] task: ffff8abafc521040 ti: ffff8abaf7e2c000 task.ti: ffff8abaf7e2c000 [1488061.346424] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.346425] RSP: 0018:ffff8abaf7e2fb78 EFLAGS: 00000246 [1488061.346426] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000b90000 [1488061.346427] RDX: ffff8aeaffa1b780 RSI: 0000000000290001 RDI: ffff8adafa7b1b00 [1488061.346429] RBP: ffff8abaf7e2fb78 R08: ffff8afb3f89b780 R09: 0000000000000000 [1488061.346430] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af3c4de2400 [1488061.346430] R13: 0005c4aa17ccfe10 R14: ffff8af51a43e000 R15: 0000000000000007 [1488061.346432] FS: 00007ea1c407c700(0000) GS:ffff8afb3f880000(0000) knlGS:0000000000000000 [1488061.346433] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.346434] CR2: 00007f8c23c55000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.346435] Call Trace: [1488061.346437] [] queued_spin_lock_slowpath+0xb/0xf [1488061.346439] [] _raw_spin_lock+0x20/0x30 [1488061.346448] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.346458] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.346496] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.346534] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.346571] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.346610] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.346612] [] ? wake_up_state+0x20/0x20 [1488061.346651] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.346653] [] kthread+0xd1/0xe0 [1488061.346654] [] ? insert_kthread_work+0x40/0x40 [1488061.346657] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.346659] [] ? insert_kthread_work+0x40/0x40 [1488061.346679] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.365383] NMI watchdog: BUG: soft lockup - CPU#25 stuck for 23s! [ptlrpcd_04_04:54208] [1488061.365410] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.365418] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.365421] CPU: 25 PID: 54208 Comm: ptlrpcd_04_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.365421] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.365423] task: ffff8aaba9656180 ti: ffff8abaf9210000 task.ti: ffff8abaf9210000 [1488061.365426] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.365427] RSP: 0018:ffff8abaf9213b78 EFLAGS: 00000246 [1488061.365428] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000c90000 [1488061.365429] RDX: ffff8aaaffa9b780 RSI: 0000000000a10001 RDI: ffff8adafa7b1b00 [1488061.365430] RBP: ffff8abaf9213b78 R08: ffff8acaffadb780 R09: 0000000000000000 [1488061.365431] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abddebabc00 [1488061.365432] R13: 0005c4aa17c49cd0 R14: ffff8abe5b605100 R15: 0000000000000007 [1488061.365434] FS: 00007ea1c487d700(0000) GS:ffff8acaffac0000(0000) knlGS:0000000000000000 [1488061.365434] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.365435] CR2: 00007ee0c35fc02a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.365436] Call Trace: [1488061.365439] [] queued_spin_lock_slowpath+0xb/0xf [1488061.365441] [] _raw_spin_lock+0x20/0x30 [1488061.365449] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.365460] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.365498] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.365536] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.365573] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.365611] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.365614] [] ? wake_up_state+0x20/0x20 [1488061.365652] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.365654] [] kthread+0xd1/0xe0 [1488061.365656] [] ? insert_kthread_work+0x40/0x40 [1488061.365658] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.365660] [] ? insert_kthread_work+0x40/0x40 [1488061.365681] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.373378] NMI watchdog: BUG: soft lockup - CPU#26 stuck for 23s! [ptlrpcd_01_00:54168] [1488061.373404] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.373411] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.373414] CPU: 26 PID: 54168 Comm: ptlrpcd_01_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.373415] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.373416] task: ffff8abaf6341040 ti: ffff8abaf9b5c000 task.ti: ffff8abaf9b5c000 [1488061.373419] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488061.373420] RSP: 0018:ffff8abaf9b5fb78 EFLAGS: 00000246 [1488061.373421] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000d10000 [1488061.373422] RDX: ffff8abaffb1b780 RSI: 0000000001310001 RDI: ffff8adafa7b1b00 [1488061.373422] RBP: ffff8abaf9b5fb78 R08: ffff8a9affadb780 R09: 0000000000000000 [1488061.373423] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8d3fec2800 [1488061.373424] R13: 0005c4aa17923260 R14: ffff8a8fb02e0000 R15: 0000000000000007 [1488061.373426] FS: 00007ea1d489d700(0000) GS:ffff8a9affac0000(0000) knlGS:0000000000000000 [1488061.373427] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.373428] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.373428] Call Trace: [1488061.373431] [] queued_spin_lock_slowpath+0xb/0xf [1488061.373433] [] _raw_spin_lock+0x20/0x30 [1488061.373440] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.373451] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.373489] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.373526] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.373564] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.373602] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.373604] [] ? wake_up_state+0x20/0x20 [1488061.373643] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.373645] [] kthread+0xd1/0xe0 [1488061.373646] [] ? insert_kthread_work+0x40/0x40 [1488061.373648] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.373650] [] ? insert_kthread_work+0x40/0x40 [1488061.373670] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488061.409385] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 22s! [ptlrpcd_03_06:54198] [1488061.409410] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.409417] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.409419] CPU: 30 PID: 54198 Comm: ptlrpcd_03_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.409420] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.409421] task: ffff8abaf63930c0 ti: ffff8abaf8b48000 task.ti: ffff8abaf8b48000 [1488061.409424] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.409424] RSP: 0018:ffff8abaf8b4bb78 EFLAGS: 00000246 [1488061.409425] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000f10000 [1488061.409426] RDX: ffff8a9affb1b780 RSI: 0000000001110001 RDI: ffff8adafa7b1b00 [1488061.409427] RBP: ffff8abaf8b4bb78 R08: ffff8abaffadb780 R09: 0000000000000000 [1488061.409427] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab49667aa00 [1488061.409428] R13: 0005c4aa17cc2790 R14: ffff8ab496640300 R15: 0000000000000007 [1488061.409429] FS: 00007ea1be070700(0000) GS:ffff8abaffac0000(0000) knlGS:0000000000000000 [1488061.409430] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.409431] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.409431] Call Trace: [1488061.409433] [] queued_spin_lock_slowpath+0xb/0xf [1488061.409435] [] _raw_spin_lock+0x20/0x30 [1488061.409444] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.409454] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.409492] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.409529] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.409566] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.409605] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.409607] [] ? wake_up_state+0x20/0x20 [1488061.409645] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.409646] [] kthread+0xd1/0xe0 [1488061.409648] [] ? insert_kthread_work+0x40/0x40 [1488061.409650] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.409651] [] ? insert_kthread_work+0x40/0x40 [1488061.409671] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.419381] NMI watchdog: BUG: soft lockup - CPU#31 stuck for 23s! [ptlrpcd_07_07:54247] [1488061.419409] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.419416] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.419419] CPU: 31 PID: 54247 Comm: ptlrpcd_07_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.419420] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.419421] task: ffff8abafc5230c0 ti: ffff8abaeeefc000 task.ti: ffff8abaeeefc000 [1488061.419424] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.419425] RSP: 0018:ffff8abaeeeffb78 EFLAGS: 00000246 [1488061.419426] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000f90000 [1488061.419427] RDX: ffff8a8aff45b780 RSI: 0000000002410001 RDI: ffff8adafa7b1b00 [1488061.419428] RBP: ffff8abaeeeffb78 R08: ffff8afb3f8db780 R09: 0000000000000000 [1488061.419429] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abccdbd9c00 [1488061.419430] R13: 0005c4aa17cd0290 R14: ffff8af2df98b300 R15: 0000000000000007 [1488061.419432] FS: 00007ea1b7863700(0000) GS:ffff8afb3f8c0000(0000) knlGS:0000000000000000 [1488061.419433] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.419434] CR2: 00007ec8efbd7ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.419435] Call Trace: [1488061.419437] [] queued_spin_lock_slowpath+0xb/0xf [1488061.419439] [] _raw_spin_lock+0x20/0x30 [1488061.419447] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.419458] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.419496] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.419534] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.419571] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.419610] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.419612] [] ? wake_up_state+0x20/0x20 [1488061.419650] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.419653] [] kthread+0xd1/0xe0 [1488061.419654] [] ? insert_kthread_work+0x40/0x40 [1488061.419656] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.419658] [] ? insert_kthread_work+0x40/0x40 [1488061.419678] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.429388] NMI watchdog: BUG: soft lockup - CPU#32 stuck for 23s! [ptlrpcd_00_10:54166] [1488061.429415] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.429422] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.429425] CPU: 32 PID: 54166 Comm: ptlrpcd_00_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.429426] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.429427] task: ffff8abaf815e180 ti: ffff8abafcb7c000 task.ti: ffff8abafcb7c000 [1488061.429430] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.429431] RSP: 0018:ffff8abafcb7fb78 EFLAGS: 00000246 [1488061.429432] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001010000 [1488061.429433] RDX: ffff8adaffb1b780 RSI: 0000000001190001 RDI: ffff8adafa7b1b00 [1488061.429434] RBP: ffff8abafcb7fb78 R08: ffff8a8aff31b780 R09: 0000000000000000 [1488061.429434] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a93f2d4b600 [1488061.429436] R13: 0005c4aa1878c5d0 R14: ffff8a9839221b00 R15: 0000000000000007 [1488061.429437] FS: 00007ea1c8885700(0000) GS:ffff8a8aff300000(0000) knlGS:0000000000000000 [1488061.429438] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.429439] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.429439] Call Trace: [1488061.429442] [] queued_spin_lock_slowpath+0xb/0xf [1488061.429444] [] _raw_spin_lock+0x20/0x30 [1488061.429451] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.429462] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.429500] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.429537] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.429574] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.429612] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.429614] [] ? wake_up_state+0x20/0x20 [1488061.429652] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.429654] [] kthread+0xd1/0xe0 [1488061.429655] [] ? insert_kthread_work+0x40/0x40 [1488061.429657] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.429659] [] ? insert_kthread_work+0x40/0x40 [1488061.429679] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.438387] NMI watchdog: BUG: soft lockup - CPU#33 stuck for 23s! [ptlrpcd_04_06:54210] [1488061.438415] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.438422] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.438425] CPU: 33 PID: 54210 Comm: ptlrpcd_04_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.438426] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.438427] task: ffff8abaf82c9040 ti: ffff8abaeee74000 task.ti: ffff8abaeee74000 [1488061.438430] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.438431] RSP: 0018:ffff8abaeee77b78 EFLAGS: 00000246 [1488061.438432] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001090000 [1488061.438433] RDX: ffff8afb3fa9b780 RSI: 0000000002b90001 RDI: ffff8adafa7b1b00 [1488061.438434] RBP: ffff8abaeee77b78 R08: ffff8acaffb1b780 R09: 0000000000000000 [1488061.438435] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abc1b7d1400 [1488061.438436] R13: 0005c4aa17c594d0 R14: ffff8acaa76d0c00 R15: 0000000000000007 [1488061.438438] FS: 00007ea1d68a1700(0000) GS:ffff8acaffb00000(0000) knlGS:0000000000000000 [1488061.438439] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.438440] CR2: 00007f8c27d6e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.438441] Call Trace: [1488061.438443] [] queued_spin_lock_slowpath+0xb/0xf [1488061.438445] [] _raw_spin_lock+0x20/0x30 [1488061.438453] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.438464] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.438502] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.438540] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.438577] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.438616] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.438618] [] ? wake_up_state+0x20/0x20 [1488061.438657] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.438658] [] kthread+0xd1/0xe0 [1488061.438660] [] ? insert_kthread_work+0x40/0x40 [1488061.438662] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.438664] [] ? insert_kthread_work+0x40/0x40 [1488061.438685] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.446387] NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [ptlrpcd_01_04:54172] [1488061.446414] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.446421] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.446424] CPU: 34 PID: 54172 Comm: ptlrpcd_01_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.446424] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.446426] task: ffff8abaf6345140 ti: ffff8abaf8ea4000 task.ti: ffff8abaf8ea4000 [1488061.446429] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.446430] RSP: 0018:ffff8abaf8ea7b78 EFLAGS: 00000246 [1488061.446430] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001110000 [1488061.446431] RDX: ffff8abaffbdb780 RSI: 0000000001f10000 RDI: ffff8adafa7b1b00 [1488061.446432] RBP: ffff8abaf8ea7b78 R08: ffff8a9affb1b780 R09: 0000000000000000 [1488061.446433] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8e12bde000 [1488061.446434] R13: 0005c4aa179e4750 R14: ffff8a9735b4bc00 R15: 0000000000000007 [1488061.446436] FS: 00007ea1dc8ad700(0000) GS:ffff8a9affb00000(0000) knlGS:0000000000000000 [1488061.446436] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.446437] CR2: 00007ef14000a005 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.446438] Call Trace: [1488061.446440] [] queued_spin_lock_slowpath+0xb/0xf [1488061.446442] [] _raw_spin_lock+0x20/0x30 [1488061.446450] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.446460] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.446498] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.446535] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.446572] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.446610] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.446613] [] ? wake_up_state+0x20/0x20 [1488061.446651] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.446652] [] kthread+0xd1/0xe0 [1488061.446654] [] ? insert_kthread_work+0x40/0x40 [1488061.446656] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.446658] [] ? insert_kthread_work+0x40/0x40 [1488061.446678] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.465387] NMI watchdog: BUG: soft lockup - CPU#36 stuck for 23s! [ptlrpcd_02_06:54186] [1488061.465412] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.465419] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.465421] CPU: 36 PID: 54186 Comm: ptlrpcd_02_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.465422] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.465423] task: ffff8abaee645140 ti: ffff8abaf62fc000 task.ti: ffff8abaf62fc000 [1488061.465426] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.465426] RSP: 0018:ffff8abaf62ffb78 EFLAGS: 00000246 [1488061.465427] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001210000 [1488061.465428] RDX: ffff8abaffc5b780 RSI: 0000000002710001 RDI: ffff8adafa7b1b00 [1488061.465429] RBP: ffff8abaf62ffb78 R08: ffff8aaaffb1b780 R09: 0000000000000000 [1488061.465430] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa90e867400 [1488061.465431] R13: 0005c4aa17cccae0 R14: ffff8a9dc242e600 R15: 0000000000000007 [1488061.465432] FS: 00007ea1c407c700(0000) GS:ffff8aaaffb00000(0000) knlGS:0000000000000000 [1488061.465433] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.465434] CR2: 00007f04ff0f6000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.465435] Call Trace: [1488061.465437] [] queued_spin_lock_slowpath+0xb/0xf [1488061.465439] [] _raw_spin_lock+0x20/0x30 [1488061.465446] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.465457] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.465495] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.465532] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.465568] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.465607] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.465609] [] ? wake_up_state+0x20/0x20 [1488061.465647] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.465649] [] kthread+0xd1/0xe0 [1488061.465650] [] ? insert_kthread_work+0x40/0x40 [1488061.465652] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.465654] [] ? insert_kthread_work+0x40/0x40 [1488061.465673] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.473388] NMI watchdog: BUG: soft lockup - CPU#37 stuck for 23s! [ptlrpcd_06_05:54233] [1488061.473417] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.473424] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.473427] CPU: 37 PID: 54233 Comm: ptlrpcd_06_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.473428] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.473429] task: ffff8abaf47830c0 ti: ffff8abaf6a84000 task.ti: ffff8abaf6a84000 [1488061.473432] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.473433] RSP: 0018:ffff8abaf6a87b78 EFLAGS: 00000246 [1488061.473434] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001290000 [1488061.473435] RDX: ffff8afb3fa5b780 RSI: 0000000002790001 RDI: ffff8adafa7b1b00 [1488061.473436] RBP: ffff8abaf6a87b78 R08: ffff8aeaffb1b780 R09: 0000000000000000 [1488061.473437] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ae3e3163800 [1488061.473438] R13: 0005c4aa17d361c0 R14: ffff8ae78ad31500 R15: 0000000000000007 [1488061.473440] FS: 00007ea1bb06a700(0000) GS:ffff8aeaffb00000(0000) knlGS:0000000000000000 [1488061.473441] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.473442] CR2: 00007ee0c2bf402a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.473443] Call Trace: [1488061.473445] [] queued_spin_lock_slowpath+0xb/0xf [1488061.473447] [] _raw_spin_lock+0x20/0x30 [1488061.473455] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.473466] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.473504] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.473542] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.473579] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.473618] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.473620] [] ? wake_up_state+0x20/0x20 [1488061.473659] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.473661] [] kthread+0xd1/0xe0 [1488061.473663] [] ? insert_kthread_work+0x40/0x40 [1488061.473665] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.473667] [] ? insert_kthread_work+0x40/0x40 [1488061.473687] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.482385] NMI watchdog: BUG: soft lockup - CPU#38 stuck for 23s! [ptlrpcd_03_08:54200] [1488061.482412] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.482419] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.482422] CPU: 38 PID: 54200 Comm: ptlrpcd_03_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.482422] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.482423] task: ffff8abaf6395140 ti: ffff8abaf23a8000 task.ti: ffff8abaf23a8000 [1488061.482426] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.482427] RSP: 0018:ffff8abaf23abb78 EFLAGS: 00000246 [1488061.482428] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001310000 [1488061.482429] RDX: ffff8adaffa1b780 RSI: 0000000000190001 RDI: ffff8adafa7b1b00 [1488061.482430] RBP: ffff8abaf23abb78 R08: ffff8abaffb1b780 R09: 0000000000000000 [1488061.482431] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab496651e00 [1488061.482431] R13: 0005c4aa17cc17c0 R14: ffff8aac13384500 R15: 0000000000000007 [1488061.482433] FS: 00007ea1c0074700(0000) GS:ffff8abaffb00000(0000) knlGS:0000000000000000 [1488061.482434] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.482435] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.482436] Call Trace: [1488061.482437] [] queued_spin_lock_slowpath+0xb/0xf [1488061.482439] [] _raw_spin_lock+0x20/0x30 [1488061.482447] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.482458] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.482496] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.482533] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.482570] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.482609] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.482611] [] ? wake_up_state+0x20/0x20 [1488061.482649] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.482651] [] kthread+0xd1/0xe0 [1488061.482653] [] ? insert_kthread_work+0x40/0x40 [1488061.482654] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.482657] [] ? insert_kthread_work+0x40/0x40 [1488061.482676] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.492387] NMI watchdog: BUG: soft lockup - CPU#39 stuck for 23s! [ptlrpcd_07_08:54248] [1488061.492415] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.492422] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.492425] CPU: 39 PID: 54248 Comm: ptlrpcd_07_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.492426] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.492427] task: ffff8abafc524100 ti: ffff8abaf97b8000 task.ti: ffff8abaf97b8000 [1488061.492431] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.492432] RSP: 0018:ffff8abaf97bbb78 EFLAGS: 00000246 [1488061.492432] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001390000 [1488061.492434] RDX: ffff8aaaffcdb780 RSI: 0000000002e10001 RDI: ffff8adafa7b1b00 [1488061.492435] RBP: ffff8abaf97bbb78 R08: ffff8afb3f91b780 R09: 0000000000000000 [1488061.492435] R10: 0000000000000000 R11: 000000000000000f R12: ffff8afadee70a00 [1488061.492437] R13: 0005c4aa17c56a70 R14: ffff8aef092d3900 R15: 0000000000000007 [1488061.492438] FS: 00007ea1ca889700(0000) GS:ffff8afb3f900000(0000) knlGS:0000000000000000 [1488061.492439] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.492440] CR2: 00007eea7f033bc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.492441] Call Trace: [1488061.492444] [] queued_spin_lock_slowpath+0xb/0xf [1488061.492446] [] _raw_spin_lock+0x20/0x30 [1488061.492454] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.492464] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.492503] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.492541] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.492578] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.492616] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.492619] [] ? wake_up_state+0x20/0x20 [1488061.492657] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.492659] [] kthread+0xd1/0xe0 [1488061.492661] [] ? insert_kthread_work+0x40/0x40 [1488061.492663] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.492665] [] ? insert_kthread_work+0x40/0x40 [1488061.492685] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.502388] NMI watchdog: BUG: soft lockup - CPU#40 stuck for 23s! [ptlrpcd_00_09:54165] [1488061.502416] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.502423] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.502426] CPU: 40 PID: 54165 Comm: ptlrpcd_00_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.502426] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.502428] task: ffff8abaf815d140 ti: ffff8abaf897c000 task.ti: ffff8abaf897c000 [1488061.502431] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.502432] RSP: 0018:ffff8abaf897fb78 EFLAGS: 00000246 [1488061.502433] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001410000 [1488061.502433] RDX: ffff8a8aff21b780 RSI: 0000000000010001 RDI: ffff8adafa7b1b00 [1488061.502434] RBP: ffff8abaf897fb78 R08: ffff8a8aff35b780 R09: 0000000000000000 [1488061.502435] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af7934f2400 [1488061.502436] R13: 0005c4aa1879f630 R14: ffff8a9016832700 R15: 0000000000000007 [1488061.502438] FS: 00007f1993824740(0000) GS:ffff8a8aff340000(0000) knlGS:0000000000000000 [1488061.502438] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.502439] CR2: 00007f198a91dd70 CR3: 0000005f8433a000 CR4: 00000000003407e0 [1488061.502440] Call Trace: [1488061.502442] [] queued_spin_lock_slowpath+0xb/0xf [1488061.502444] [] _raw_spin_lock+0x20/0x30 [1488061.502452] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.502463] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.502501] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.502539] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.502576] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.502615] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.502617] [] ? wake_up_state+0x20/0x20 [1488061.502656] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.502658] [] kthread+0xd1/0xe0 [1488061.502660] [] ? insert_kthread_work+0x40/0x40 [1488061.502662] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.502663] [] ? insert_kthread_work+0x40/0x40 [1488061.502684] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.519389] NMI watchdog: BUG: soft lockup - CPU#42 stuck for 23s! [ptlrpcd_01_01:54169] [1488061.519415] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.519423] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.519425] CPU: 42 PID: 54169 Comm: ptlrpcd_01_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.519426] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.519427] task: ffff8abaf6342080 ti: ffff8abaf1700000 task.ti: ffff8abaf1700000 [1488061.519430] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.519431] RSP: 0018:ffff8abaf1703b78 EFLAGS: 00000246 [1488061.519432] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001510000 [1488061.519432] RDX: ffff8aaaffb9b780 RSI: 0000000001a10001 RDI: ffff8adafa7b1b00 [1488061.519433] RBP: ffff8abaf1703b78 R08: ffff8a9affb5b780 R09: 0000000000000000 [1488061.519434] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a94d1231600 [1488061.519435] R13: 0005c4aa179bd630 R14: ffff8a8ed6a73c00 R15: 0000000000000007 [1488061.519437] FS: 00007ea1b2859700(0000) GS:ffff8a9affb40000(0000) knlGS:0000000000000000 [1488061.519437] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.519438] CR2: 00007ef4a7d2d000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.519439] Call Trace: [1488061.519441] [] queued_spin_lock_slowpath+0xb/0xf [1488061.519443] [] _raw_spin_lock+0x20/0x30 [1488061.519451] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.519461] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.519499] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.519536] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.519573] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.519611] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.519613] [] ? wake_up_state+0x20/0x20 [1488061.519651] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.519653] [] kthread+0xd1/0xe0 [1488061.519655] [] ? insert_kthread_work+0x40/0x40 [1488061.519657] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.519659] [] ? insert_kthread_work+0x40/0x40 [1488061.519678] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.529389] NMI watchdog: BUG: soft lockup - CPU#43 stuck for 23s! [ptlrpcd_05_10:54226] [1488061.529417] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.529425] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.529428] CPU: 43 PID: 54226 Comm: ptlrpcd_05_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.529429] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.529430] task: ffff8abafb33b0c0 ti: ffff8abaeeef4000 task.ti: ffff8abaeeef4000 [1488061.529433] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.529435] RSP: 0018:ffff8abaeeef7b78 EFLAGS: 00000246 [1488061.529436] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001590000 [1488061.529437] RDX: ffff8afb3f8db780 RSI: 0000000000f90001 RDI: ffff8adafa7b1b00 [1488061.529438] RBP: ffff8abaeeef7b78 R08: ffff8adaffb5b780 R09: 0000000000000000 [1488061.529439] R10: 0000000000000000 R11: 000000000000000f R12: ffff8acb4b436800 [1488061.529440] R13: 0005c4aa1785f220 R14: ffff8a9421aa0900 R15: 0000000000000007 [1488061.529441] FS: 00007ea1b2859700(0000) GS:ffff8adaffb40000(0000) knlGS:0000000000000000 [1488061.529442] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.529443] CR2: 0000000002908888 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.529444] Call Trace: [1488061.529447] [] queued_spin_lock_slowpath+0xb/0xf [1488061.529449] [] _raw_spin_lock+0x20/0x30 [1488061.529456] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.529467] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.529506] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.529544] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.529581] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.529619] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.529622] [] ? wake_up_state+0x20/0x20 [1488061.529660] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.529662] [] kthread+0xd1/0xe0 [1488061.529664] [] ? insert_kthread_work+0x40/0x40 [1488061.529666] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.529668] [] ? insert_kthread_work+0x40/0x40 [1488061.529688] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.539394] NMI watchdog: BUG: soft lockup - CPU#44 stuck for 23s! [ptlrpcd_02_08:54188] [1488061.539421] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.539429] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.539431] CPU: 44 PID: 54188 Comm: ptlrpcd_02_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.539432] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.539433] task: ffff8abaebaf8000 ti: ffff8abaecba8000 task.ti: ffff8abaecba8000 [1488061.539436] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.539437] RSP: 0018:ffff8abaecbabb78 EFLAGS: 00000246 [1488061.539438] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001610000 [1488061.539439] RDX: ffff8aeaffc1b780 RSI: 0000000002290001 RDI: ffff8adafa7b1b00 [1488061.539439] RBP: ffff8abaecbabb78 R08: ffff8aaaffb5b780 R09: 0000000000000000 [1488061.539440] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9b52033000 [1488061.539441] R13: 0005c4aa17cd9280 R14: ffff8aa69eaf9800 R15: 0000000000000007 [1488061.539443] FS: 00007ea1ba068700(0000) GS:ffff8aaaffb40000(0000) knlGS:0000000000000000 [1488061.539443] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.539444] CR2: 00007edc08f4ef58 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.539445] Call Trace: [1488061.539447] [] queued_spin_lock_slowpath+0xb/0xf [1488061.539449] [] _raw_spin_lock+0x20/0x30 [1488061.539457] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.539468] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.539506] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.539545] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.539582] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.539620] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.539623] [] ? wake_up_state+0x20/0x20 [1488061.539661] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.539663] [] kthread+0xd1/0xe0 [1488061.539664] [] ? insert_kthread_work+0x40/0x40 [1488061.539667] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.539669] [] ? insert_kthread_work+0x40/0x40 [1488061.539689] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.556394] NMI watchdog: BUG: soft lockup - CPU#46 stuck for 23s! [ptlrpcd_03_07:54199] [1488061.556420] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.556427] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.556429] CPU: 46 PID: 54199 Comm: ptlrpcd_03_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.556430] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.556432] task: ffff8abaf6394100 ti: ffff8abafbfc0000 task.ti: ffff8abafbfc0000 [1488061.556435] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.556436] RSP: 0018:ffff8abafbfc3b78 EFLAGS: 00000246 [1488061.556437] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001710000 [1488061.556438] RDX: ffff8afb3f85b780 RSI: 0000000000790001 RDI: ffff8adafa7b1b00 [1488061.556439] RBP: ffff8abafbfc3b78 R08: ffff8abaffb5b780 R09: 0000000000000000 [1488061.556441] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aabdc308400 [1488061.556441] R13: 0005c4aa17d34e70 R14: ffff8ab441780900 R15: 0000000000000007 [1488061.556443] FS: 00007f1993824740(0000) GS:ffff8abaffb40000(0000) knlGS:0000000000000000 [1488061.556444] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.556445] CR2: 00007f002a73a795 CR3: 0000005f8433a000 CR4: 00000000003407e0 [1488061.556446] Call Trace: [1488061.556448] [] queued_spin_lock_slowpath+0xb/0xf [1488061.556450] [] _raw_spin_lock+0x20/0x30 [1488061.556458] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.556468] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.556506] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.556544] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.556580] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.556619] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.556621] [] ? wake_up_state+0x20/0x20 [1488061.556659] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.556661] [] kthread+0xd1/0xe0 [1488061.556662] [] ? insert_kthread_work+0x40/0x40 [1488061.556665] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.556667] [] ? insert_kthread_work+0x40/0x40 [1488061.556686] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.566393] NMI watchdog: BUG: soft lockup - CPU#47 stuck for 23s! [ptlrpcd_07_02:54242] [1488061.566421] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.566428] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.566432] CPU: 47 PID: 54242 Comm: ptlrpcd_07_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.566433] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.566434] task: ffff8abafbc7d140 ti: ffff8abafcdb4000 task.ti: ffff8abafcdb4000 [1488061.566437] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.566438] RSP: 0018:ffff8abafcdb7b78 EFLAGS: 00000246 [1488061.566439] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001790000 [1488061.566440] RDX: ffff8abaffa1b780 RSI: 0000000000310001 RDI: ffff8adafa7b1b00 [1488061.566441] RBP: ffff8abafcdb7b78 R08: ffff8afb3f95b780 R09: 0000000000000000 [1488061.566442] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af4f0abdc00 [1488061.566443] R13: 0005c4aa17bf9cb0 R14: ffff8af5fcf29e00 R15: 0000000000000007 [1488061.566445] FS: 00007ea1c9887700(0000) GS:ffff8afb3f940000(0000) knlGS:0000000000000000 [1488061.566446] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.566447] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.566448] Call Trace: [1488061.566450] [] queued_spin_lock_slowpath+0xb/0xf [1488061.566452] [] _raw_spin_lock+0x20/0x30 [1488061.566460] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.566471] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.566509] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.566547] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.566584] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.566622] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.566625] [] ? wake_up_state+0x20/0x20 [1488061.566663] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.566665] [] kthread+0xd1/0xe0 [1488061.566667] [] ? insert_kthread_work+0x40/0x40 [1488061.566669] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.566671] [] ? insert_kthread_work+0x40/0x40 [1488061.566691] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.584397] NMI watchdog: BUG: soft lockup - CPU#49 stuck for 23s! [ptlrpcd_04_08:54212] [1488061.584424] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.584431] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.584434] CPU: 49 PID: 54212 Comm: ptlrpcd_04_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.584435] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.584436] task: ffff8abaf82cb0c0 ti: ffff8abaedea0000 task.ti: ffff8abaedea0000 [1488061.584439] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.584440] RSP: 0018:ffff8abaedea3b78 EFLAGS: 00000246 [1488061.584441] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001890000 [1488061.584442] RDX: ffff8afb3f95b780 RSI: 0000000001790001 RDI: ffff8adafa7b1b00 [1488061.584443] RBP: ffff8abaedea3b78 R08: ffff8acaffb9b780 R09: 0000000000000000 [1488061.584444] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abf38b6d000 [1488061.584445] R13: 0005c4aa17c5ad80 R14: ffff8acae8391200 R15: 0000000000000007 [1488061.584447] FS: 00007ea1de0b0700(0000) GS:ffff8acaffb80000(0000) knlGS:0000000000000000 [1488061.584448] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.584449] CR2: 00007ec111dde00a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.584450] Call Trace: [1488061.584452] [] queued_spin_lock_slowpath+0xb/0xf [1488061.584454] [] _raw_spin_lock+0x20/0x30 [1488061.584462] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.584473] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.584511] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.584548] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.584586] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.584624] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.584626] [] ? wake_up_state+0x20/0x20 [1488061.584665] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.584667] [] kthread+0xd1/0xe0 [1488061.584669] [] ? insert_kthread_work+0x40/0x40 [1488061.584671] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.584673] [] ? insert_kthread_work+0x40/0x40 [1488061.584692] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.611396] NMI watchdog: BUG: soft lockup - CPU#52 stuck for 22s! [ptlrpcd_02_09:54189] [1488061.611423] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.611431] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.611433] CPU: 52 PID: 54189 Comm: ptlrpcd_02_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.611434] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.611435] task: ffff8abaebaf9040 ti: ffff8abaf7e44000 task.ti: ffff8abaf7e44000 [1488061.611438] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.611439] RSP: 0018:ffff8abaf7e47b78 EFLAGS: 00000246 [1488061.611440] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001a10000 [1488061.611441] RDX: ffff8aaaffa1b780 RSI: 0000000000210001 RDI: ffff8adafa7b1b00 [1488061.611442] RBP: ffff8abaf7e47b78 R08: ffff8aaaffb9b780 R09: 0000000000000000 [1488061.611443] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa83ad31e00 [1488061.611444] R13: 0005c4aa17cfec30 R14: ffff8a9bcba58f00 R15: 0000000000000007 [1488061.611446] FS: 00007ea1d509e700(0000) GS:ffff8aaaffb80000(0000) knlGS:0000000000000000 [1488061.611447] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.611448] CR2: 00007eea7f02bbc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.611449] Call Trace: [1488061.611451] [] queued_spin_lock_slowpath+0xb/0xf [1488061.611453] [] _raw_spin_lock+0x20/0x30 [1488061.611461] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.611472] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.611509] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.611547] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.611583] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.611623] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.611626] [] ? wake_up_state+0x20/0x20 [1488061.611664] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.611665] [] kthread+0xd1/0xe0 [1488061.611667] [] ? insert_kthread_work+0x40/0x40 [1488061.611669] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.611671] [] ? insert_kthread_work+0x40/0x40 [1488061.611691] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.628395] NMI watchdog: BUG: soft lockup - CPU#54 stuck for 22s! [ptlrpcd_03_04:54196] [1488061.628421] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.628429] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.628431] CPU: 54 PID: 54196 Comm: ptlrpcd_03_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.628432] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.628433] task: ffff8abaf6391040 ti: ffff8aba98688000 task.ti: ffff8aba98688000 [1488061.628436] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.628437] RSP: 0018:ffff8aba9868bb78 EFLAGS: 00000246 [1488061.628438] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001b10000 [1488061.628439] RDX: ffff8aeaffc9b780 RSI: 0000000002a90001 RDI: ffff8adafa7b1b00 [1488061.628440] RBP: ffff8aba9868bb78 R08: ffff8abaffb9b780 R09: 0000000000000000 [1488061.628441] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab8d9dd2e00 [1488061.628442] R13: 0005c4aa17cb5c70 R14: ffff8ab98a500c00 R15: 0000000000000007 [1488061.628444] FS: 00007ea1c7082700(0000) GS:ffff8abaffb80000(0000) knlGS:0000000000000000 [1488061.628445] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.628446] CR2: 00007f3dc9661978 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.628447] Call Trace: [1488061.628449] [] queued_spin_lock_slowpath+0xb/0xf [1488061.628450] [] _raw_spin_lock+0x20/0x30 [1488061.628458] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.628469] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.628507] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.628545] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.628582] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.628620] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.628623] [] ? wake_up_state+0x20/0x20 [1488061.628661] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.628663] [] kthread+0xd1/0xe0 [1488061.628665] [] ? insert_kthread_work+0x40/0x40 [1488061.628666] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.628668] [] ? insert_kthread_work+0x40/0x40 [1488061.628689] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.637399] NMI watchdog: BUG: soft lockup - CPU#55 stuck for 22s! [ptlrpcd_07_09:54249] [1488061.637427] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.637434] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.637437] CPU: 55 PID: 54249 Comm: ptlrpcd_07_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.637437] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.637439] task: ffff8abafc525140 ti: ffff8abaee6c0000 task.ti: ffff8abaee6c0000 [1488061.637442] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.637443] RSP: 0018:ffff8abaee6c3b78 EFLAGS: 00000246 [1488061.637443] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001b90000 [1488061.637445] RDX: ffff8aeaffbdb780 RSI: 0000000001e90001 RDI: ffff8adafa7b1b00 [1488061.637445] RBP: ffff8abaee6c3b78 R08: ffff8afb3f99b780 R09: 0000000000000000 [1488061.637446] R10: 0000000000000000 R11: 000000000000000f R12: ffff8afaaa72c600 [1488061.637448] R13: 0005c4aa17cce010 R14: ffff8af51a438000 R15: 0000000000000007 [1488061.637449] FS: 00007ea1cf893700(0000) GS:ffff8afb3f980000(0000) knlGS:0000000000000000 [1488061.637450] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.637451] CR2: 00007f6e7e5bf590 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.637452] Call Trace: [1488061.637454] [] queued_spin_lock_slowpath+0xb/0xf [1488061.637457] [] _raw_spin_lock+0x20/0x30 [1488061.637464] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.637475] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.637513] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.637551] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.637588] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.637626] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.637629] [] ? wake_up_state+0x20/0x20 [1488061.637667] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.637669] [] kthread+0xd1/0xe0 [1488061.637671] [] ? insert_kthread_work+0x40/0x40 [1488061.637673] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.637675] [] ? insert_kthread_work+0x40/0x40 [1488061.637695] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.655401] NMI watchdog: BUG: soft lockup - CPU#57 stuck for 22s! [ptlrpcd_04_09:54213] [1488061.655428] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.655436] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.655438] CPU: 57 PID: 54213 Comm: ptlrpcd_04_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.655439] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.655440] task: ffff8abaf82cc100 ti: ffff8abaf69e0000 task.ti: ffff8abaf69e0000 [1488061.655443] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.655444] RSP: 0018:ffff8abaf69e3b78 EFLAGS: 00000246 [1488061.655445] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001c90000 [1488061.655446] RDX: ffff8acaffb5b780 RSI: 0000000001490001 RDI: ffff8adafa7b1b00 [1488061.655447] RBP: ffff8abaf69e3b78 R08: ffff8acaffbdb780 R09: 0000000000000000 [1488061.655448] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac421387200 [1488061.655449] R13: 0005c4aa17cc5280 R14: ffff8ac42130c800 R15: 0000000000000007 [1488061.655451] FS: 00007ea1bb06a700(0000) GS:ffff8acaffbc0000(0000) knlGS:0000000000000000 [1488061.655452] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.655453] CR2: 00007f8c23c55000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.655454] Call Trace: [1488061.655455] [] queued_spin_lock_slowpath+0xb/0xf [1488061.655458] [] _raw_spin_lock+0x20/0x30 [1488061.655466] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.655476] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.655514] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.655516] [] ? del_timer_sync+0x52/0x60 [1488061.655553] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.655591] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.655629] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.655631] [] ? wake_up_state+0x20/0x20 [1488061.655670] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.655671] [] kthread+0xd1/0xe0 [1488061.655673] [] ? insert_kthread_work+0x40/0x40 [1488061.655675] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.655677] [] ? insert_kthread_work+0x40/0x40 [1488061.655697] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.684403] NMI watchdog: BUG: soft lockup - CPU#60 stuck for 22s! [ptlrpcd_02_04:54184] [1488061.684429] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.684436] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.684439] CPU: 60 PID: 54184 Comm: ptlrpcd_02_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.684440] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.684441] task: ffff8abaee6430c0 ti: ffff8abaed324000 task.ti: ffff8abaed324000 [1488061.684444] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.684445] RSP: 0018:ffff8abaed327b78 EFLAGS: 00000246 [1488061.684446] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001e10000 [1488061.684447] RDX: ffff8aeaffc5b780 RSI: 0000000002690001 RDI: ffff8adafa7b1b00 [1488061.684448] RBP: ffff8abaed327b78 R08: ffff8aaaffbdb780 R09: 0000000000000000 [1488061.684449] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa8abf4ac00 [1488061.684450] R13: 0005c4aa17cdb290 R14: ffff8a9c8c644b00 R15: 0000000000000007 [1488061.684451] FS: 00007ea1d1897700(0000) GS:ffff8aaaffbc0000(0000) knlGS:0000000000000000 [1488061.684452] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.684453] CR2: 00007f8c1846c000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.684454] Call Trace: [1488061.684456] [] queued_spin_lock_slowpath+0xb/0xf [1488061.684458] [] _raw_spin_lock+0x20/0x30 [1488061.684466] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.684476] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.684514] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.684551] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.684588] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.684626] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.684628] [] ? wake_up_state+0x20/0x20 [1488061.684666] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.684668] [] kthread+0xd1/0xe0 [1488061.684670] [] ? insert_kthread_work+0x40/0x40 [1488061.684672] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.684674] [] ? insert_kthread_work+0x40/0x40 [1488061.684694] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.701401] NMI watchdog: BUG: soft lockup - CPU#62 stuck for 22s! [ptlrpcd_03_09:54201] [1488061.701427] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.701434] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.701437] CPU: 62 PID: 54201 Comm: ptlrpcd_03_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.701438] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.701439] task: ffff8abaf6396180 ti: ffff8abaf8fd0000 task.ti: ffff8abaf8fd0000 [1488061.701442] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.701443] RSP: 0018:ffff8abaf8fd3b78 EFLAGS: 00000246 [1488061.701444] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001f10000 [1488061.701445] RDX: ffff8abaffc9b780 RSI: 0000000002b10001 RDI: ffff8adafa7b1b00 [1488061.701446] RBP: ffff8abaf8fd3b78 R08: ffff8abaffbdb780 R09: 0000000000000000 [1488061.701447] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab759a35400 [1488061.701448] R13: 0005c4aa17d30930 R14: ffff8ab7c3442100 R15: 0000000000000007 [1488061.701450] FS: 00007ea1dc0ac700(0000) GS:ffff8abaffbc0000(0000) knlGS:0000000000000000 [1488061.701450] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.701451] CR2: 00007f8c2c495000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.701452] Call Trace: [1488061.701455] [] queued_spin_lock_slowpath+0xb/0xf [1488061.701457] [] _raw_spin_lock+0x20/0x30 [1488061.701464] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.701475] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.701513] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.701550] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.701587] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.701625] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.701627] [] ? wake_up_state+0x20/0x20 [1488061.701666] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.701667] [] kthread+0xd1/0xe0 [1488061.701669] [] ? insert_kthread_work+0x40/0x40 [1488061.701672] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.701673] [] ? insert_kthread_work+0x40/0x40 [1488061.701693] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.721400] NMI watchdog: BUG: soft lockup - CPU#64 stuck for 22s! [ptlrpcd_00_06:54162] [1488061.721427] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.721435] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.721437] CPU: 64 PID: 54162 Comm: ptlrpcd_00_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.721438] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.721439] task: ffff8abaf815a080 ti: ffff8abafcf18000 task.ti: ffff8abafcf18000 [1488061.721442] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.721444] RSP: 0018:ffff8abafcf1bb78 EFLAGS: 00000246 [1488061.721445] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002010000 [1488061.721446] RDX: ffff8a9affa1b780 RSI: 0000000000110001 RDI: ffff8adafa7b1b00 [1488061.721447] RBP: ffff8abafcf1bb78 R08: ffff8a8aff41b780 R09: 0000000000000000 [1488061.721448] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c3200 [1488061.721449] R13: 0005c4aa1879ffe0 R14: ffff8a90d9b17500 R15: 0000000000000007 [1488061.721451] FS: 00007f471ea2d700(0000) GS:ffff8a8aff400000(0000) knlGS:0000000000000000 [1488061.721451] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.721453] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.721453] Call Trace: [1488061.721456] [] queued_spin_lock_slowpath+0xb/0xf [1488061.721458] [] _raw_spin_lock+0x20/0x30 [1488061.721466] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.721476] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.721514] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.721552] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.721589] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.721628] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.721630] [] ? wake_up_state+0x20/0x20 [1488061.721668] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.721670] [] kthread+0xd1/0xe0 [1488061.721672] [] ? insert_kthread_work+0x40/0x40 [1488061.721674] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.721676] [] ? insert_kthread_work+0x40/0x40 [1488061.721696] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.730401] NMI watchdog: BUG: soft lockup - CPU#65 stuck for 22s! [ptlrpcd_04_02:54206] [1488061.730429] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.730436] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.730439] CPU: 65 PID: 54206 Comm: ptlrpcd_04_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.730440] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.730441] task: ffff8aaba9654100 ti: ffff8abaed798000 task.ti: ffff8abaed798000 [1488061.730444] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.730445] RSP: 0018:ffff8abaed79bb78 EFLAGS: 00000246 [1488061.730446] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002090000 [1488061.730447] RDX: ffff8adaffc5b780 RSI: 0000000002590001 RDI: ffff8adafa7b1b00 [1488061.730448] RBP: ffff8abaed79bb78 R08: ffff8acaffc1b780 R09: 0000000000000000 [1488061.730449] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abf71fb9a00 [1488061.730449] R13: 0005c4aa17c537a0 R14: ffff8abe5b6fe300 R15: 0000000000000007 [1488061.730451] FS: 00007ea1be871700(0000) GS:ffff8acaffc00000(0000) knlGS:0000000000000000 [1488061.730452] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.730453] CR2: 00007ee8a0ba6013 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.730454] Call Trace: [1488061.730456] [] queued_spin_lock_slowpath+0xb/0xf [1488061.730458] [] _raw_spin_lock+0x20/0x30 [1488061.730466] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.730477] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.730515] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.730553] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.730590] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.730628] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.730630] [] ? wake_up_state+0x20/0x20 [1488061.730669] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.730671] [] kthread+0xd1/0xe0 [1488061.730673] [] ? insert_kthread_work+0x40/0x40 [1488061.730674] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.730676] [] ? insert_kthread_work+0x40/0x40 [1488061.730696] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.738401] NMI watchdog: BUG: soft lockup - CPU#66 stuck for 22s! [ptlrpcd_01_11:54179] [1488061.738429] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.738436] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.738439] CPU: 66 PID: 54179 Comm: ptlrpcd_01_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.738440] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.738441] task: ffff8abaf4dad140 ti: ffff8abaf8720000 task.ti: ffff8abaf8720000 [1488061.738444] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488061.738445] RSP: 0018:ffff8abaf8723b78 EFLAGS: 00000246 [1488061.738446] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002110000 [1488061.738448] RDX: ffff8a9affa5b780 RSI: 0000000000510001 RDI: ffff8adafa7b1b00 [1488061.738449] RBP: ffff8abaf8723b78 R08: ffff8a9affc1b780 R09: 0000000000000000 [1488061.738449] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a90f4378600 [1488061.738451] R13: 0005c4aa17a3c990 R14: ffff8a931021e900 R15: 0000000000000007 [1488061.738452] FS: 00007ea1cb88b700(0000) GS:ffff8a9affc00000(0000) knlGS:0000000000000000 [1488061.738453] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.738454] CR2: 00007ed1c5b95005 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.738455] Call Trace: [1488061.738457] [] queued_spin_lock_slowpath+0xb/0xf [1488061.738459] [] _raw_spin_lock+0x20/0x30 [1488061.738467] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.738478] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.738515] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.738553] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.738589] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.738628] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.738630] [] ? wake_up_state+0x20/0x20 [1488061.738669] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.738670] [] kthread+0xd1/0xe0 [1488061.738672] [] ? insert_kthread_work+0x40/0x40 [1488061.738674] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.738676] [] ? insert_kthread_work+0x40/0x40 [1488061.738696] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488061.766407] NMI watchdog: BUG: soft lockup - CPU#69 stuck for 22s! [ptlrpcd_06_06:54234] [1488061.766435] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.766442] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.766444] CPU: 69 PID: 54234 Comm: ptlrpcd_06_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.766445] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.766446] task: ffff8abaf4784100 ti: ffff8abaefaf4000 task.ti: ffff8abaefaf4000 [1488061.766449] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.766450] RSP: 0018:ffff8abaefaf7b78 EFLAGS: 00000246 [1488061.766451] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002290000 [1488061.766452] RDX: ffff8a9affb9b780 RSI: 0000000001910001 RDI: ffff8adafa7b1b00 [1488061.766453] RBP: ffff8abaefaf7b78 R08: ffff8aeaffc1b780 R09: 0000000000000000 [1488061.766454] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adccf56cc00 [1488061.766455] R13: 0005c4aa17c91e00 R14: ffff8aeaaa641200 R15: 0000000000000007 [1488061.766457] FS: 00007ea1b7863700(0000) GS:ffff8aeaffc00000(0000) knlGS:0000000000000000 [1488061.766458] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.766459] CR2: 00007f47239c9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.766460] Call Trace: [1488061.766462] [] queued_spin_lock_slowpath+0xb/0xf [1488061.766464] [] _raw_spin_lock+0x20/0x30 [1488061.766472] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.766483] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.766521] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.766559] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.766596] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.766634] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.766636] [] ? wake_up_state+0x20/0x20 [1488061.766675] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.766677] [] kthread+0xd1/0xe0 [1488061.766679] [] ? insert_kthread_work+0x40/0x40 [1488061.766681] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.766683] [] ? insert_kthread_work+0x40/0x40 [1488061.766703] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.775406] NMI watchdog: BUG: soft lockup - CPU#70 stuck for 22s! [ptlrpcd_03_03:54195] [1488061.775434] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.775441] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.775444] CPU: 70 PID: 54195 Comm: ptlrpcd_03_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.775445] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.775446] task: ffff8abaf6390000 ti: ffff8abaf86a0000 task.ti: ffff8abaf86a0000 [1488061.775449] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.775450] RSP: 0018:ffff8abaf86a3b78 EFLAGS: 00000246 [1488061.775451] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002310000 [1488061.775452] RDX: ffff8a8aff3db780 RSI: 0000000001c10001 RDI: ffff8adafa7b1b00 [1488061.775453] RBP: ffff8abaf86a3b78 R08: ffff8abaffc1b780 R09: 0000000000000000 [1488061.775454] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abaf1e96a00 [1488061.775455] R13: 0005c4aa17d25530 R14: ffff8aafa8556000 R15: 0000000000000007 [1488061.775457] FS: 00007ea1c7082700(0000) GS:ffff8abaffc00000(0000) knlGS:0000000000000000 [1488061.775458] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.775459] CR2: 00007ebde3c2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.775460] Call Trace: [1488061.775462] [] queued_spin_lock_slowpath+0xb/0xf [1488061.775464] [] _raw_spin_lock+0x20/0x30 [1488061.775472] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.775482] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.775520] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.775558] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.775595] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.775634] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.775636] [] ? wake_up_state+0x20/0x20 [1488061.775674] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.775676] [] kthread+0xd1/0xe0 [1488061.775678] [] ? insert_kthread_work+0x40/0x40 [1488061.775680] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.775682] [] ? insert_kthread_work+0x40/0x40 [1488061.775702] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.785410] NMI watchdog: BUG: soft lockup - CPU#71 stuck for 22s! [ptlrpcd_07_11:54251] [1488061.785437] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.785444] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.785447] CPU: 71 PID: 54251 Comm: ptlrpcd_07_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.785448] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.785449] task: ffff8abafc7d8000 ti: ffff8abaf53fc000 task.ti: ffff8abaf53fc000 [1488061.785452] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.785453] RSP: 0018:ffff8abaf53ffb78 EFLAGS: 00000246 [1488061.785454] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002390000 [1488061.785455] RDX: ffff8acaffb1b780 RSI: 0000000001090001 RDI: ffff8adafa7b1b00 [1488061.785456] RBP: ffff8abaf53ffb78 R08: ffff8afb3fa1b780 R09: 0000000000000000 [1488061.785457] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af57d7fd600 [1488061.785458] R13: 0005c4aa17cd7110 R14: ffff8aec26dd7b00 R15: 0000000000000007 [1488061.785459] FS: 00007ea1d389b700(0000) GS:ffff8afb3fa00000(0000) knlGS:0000000000000000 [1488061.785460] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.785461] CR2: 00007ec111bda000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.785462] Call Trace: [1488061.785465] [] queued_spin_lock_slowpath+0xb/0xf [1488061.785467] [] _raw_spin_lock+0x20/0x30 [1488061.785474] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.785485] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.785523] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.785561] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.785597] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.785636] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.785639] [] ? wake_up_state+0x20/0x20 [1488061.785677] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.785679] [] kthread+0xd1/0xe0 [1488061.785681] [] ? insert_kthread_work+0x40/0x40 [1488061.785683] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.785685] [] ? insert_kthread_work+0x40/0x40 [1488061.785705] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.793406] NMI watchdog: BUG: soft lockup - CPU#72 stuck for 22s! [ptlrpcd_00_04:54160] [1488061.793434] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.793441] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.793444] CPU: 72 PID: 54160 Comm: ptlrpcd_00_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.793445] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.793446] task: ffff8abaf8158000 ti: ffff8abaf4a70000 task.ti: ffff8abaf4a70000 [1488061.793449] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.793450] RSP: 0018:ffff8abaf4a73b78 EFLAGS: 00000246 [1488061.793451] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002410000 [1488061.793452] RDX: ffff8aeaffadb780 RSI: 0000000000e90001 RDI: ffff8adafa7b1b00 [1488061.793453] RBP: ffff8abaf4a73b78 R08: ffff8a8aff45b780 R09: 0000000000000000 [1488061.793454] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c0200 [1488061.793455] R13: 0005c4aa1879fac0 R14: ffff8a7b02ec6f00 R15: 0000000000000007 [1488061.793457] FS: 00007ea1ce090700(0000) GS:ffff8a8aff440000(0000) knlGS:0000000000000000 [1488061.793458] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.793459] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.793460] Call Trace: [1488061.793462] [] queued_spin_lock_slowpath+0xb/0xf [1488061.793464] [] _raw_spin_lock+0x20/0x30 [1488061.793472] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.793483] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.793521] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.793559] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.793596] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.793635] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.793637] [] ? wake_up_state+0x20/0x20 [1488061.793676] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.793677] [] kthread+0xd1/0xe0 [1488061.793679] [] ? insert_kthread_work+0x40/0x40 [1488061.793681] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.793683] [] ? insert_kthread_work+0x40/0x40 [1488061.793703] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.802409] NMI watchdog: BUG: soft lockup - CPU#73 stuck for 22s! [ptlrpcd_04_01:54205] [1488061.802436] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.802444] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.802447] CPU: 73 PID: 54205 Comm: ptlrpcd_04_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.802448] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.802449] task: ffff8aaba96530c0 ti: ffff8abaf9410000 task.ti: ffff8abaf9410000 [1488061.802452] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.802453] RSP: 0018:ffff8abaf9413b78 EFLAGS: 00000246 [1488061.802454] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002490000 [1488061.802455] RDX: ffff8abaffa9b780 RSI: 0000000000b10001 RDI: ffff8adafa7b1b00 [1488061.802456] RBP: ffff8abaf9413b78 R08: ffff8acaffc5b780 R09: 0000000000000000 [1488061.802457] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aca6477ea00 [1488061.802458] R13: 0005c4aa17cb5950 R14: ffff8acaace2c500 R15: 0000000000000007 [1488061.802460] FS: 00007ea1c8084700(0000) GS:ffff8acaffc40000(0000) knlGS:0000000000000000 [1488061.802460] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.802462] CR2: 00007fab9fc94ac0 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.802462] Call Trace: [1488061.802464] [] queued_spin_lock_slowpath+0xb/0xf [1488061.802466] [] _raw_spin_lock+0x20/0x30 [1488061.802475] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.802485] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.802523] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.802561] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.802598] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.802637] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.802639] [] ? wake_up_state+0x20/0x20 [1488061.802677] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.802679] [] kthread+0xd1/0xe0 [1488061.802680] [] ? insert_kthread_work+0x40/0x40 [1488061.802683] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.802685] [] ? insert_kthread_work+0x40/0x40 [1488061.802704] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.821413] NMI watchdog: BUG: soft lockup - CPU#75 stuck for 22s! [ptlrpcd_05_11:54227] [1488061.821440] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.821447] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.821450] CPU: 75 PID: 54227 Comm: ptlrpcd_05_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.821451] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.821452] task: ffff8abafb33c100 ti: ffff8abafc494000 task.ti: ffff8abafc494000 [1488061.821455] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.821456] RSP: 0018:ffff8abafc497b78 EFLAGS: 00000246 [1488061.821457] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002590000 [1488061.821458] RDX: ffff8adaffa9b780 RSI: 0000000000990001 RDI: ffff8adafa7b1b00 [1488061.821459] RBP: ffff8abafc497b78 R08: ffff8adaffc5b780 R09: 0000000000000000 [1488061.821460] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad07cfce000 [1488061.821461] R13: 0005c4aa179b3300 R14: ffff8ada99cc5d00 R15: 0000000000000007 [1488061.821463] FS: 00007ea1b2859700(0000) GS:ffff8adaffc40000(0000) knlGS:0000000000000000 [1488061.821463] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.821464] CR2: 00007f8c11e1b000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.821465] Call Trace: [1488061.821468] [] queued_spin_lock_slowpath+0xb/0xf [1488061.821470] [] _raw_spin_lock+0x20/0x30 [1488061.821477] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.821488] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.821526] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.821564] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.821601] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.821640] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.821642] [] ? wake_up_state+0x20/0x20 [1488061.821680] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.821682] [] kthread+0xd1/0xe0 [1488061.821684] [] ? insert_kthread_work+0x40/0x40 [1488061.821686] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.821688] [] ? insert_kthread_work+0x40/0x40 [1488061.821708] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.829411] NMI watchdog: BUG: soft lockup - CPU#76 stuck for 22s! [ptlrpcd_02_11:54191] [1488061.829438] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.829445] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.829448] CPU: 76 PID: 54191 Comm: ptlrpcd_02_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.829449] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.829450] task: ffff8abaebafb0c0 ti: ffff8abaeef54000 task.ti: ffff8abaeef54000 [1488061.829453] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.829455] RSP: 0018:ffff8abaeef57b78 EFLAGS: 00000246 [1488061.829456] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002610000 [1488061.829457] RDX: ffff8acaffa1b780 RSI: 0000000000090001 RDI: ffff8adafa7b1b00 [1488061.829457] RBP: ffff8abaeef57b78 R08: ffff8aaaffc5b780 R09: 0000000000000000 [1488061.829458] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa690382600 [1488061.829459] R13: 0005c4aa17cfa010 R14: ffff8adf00a89e00 R15: 0000000000000007 [1488061.829461] FS: 00007ea1b585f700(0000) GS:ffff8aaaffc40000(0000) knlGS:0000000000000000 [1488061.829462] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.829463] CR2: 00007f8c1d455000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.829464] Call Trace: [1488061.829466] [] queued_spin_lock_slowpath+0xb/0xf [1488061.829468] [] _raw_spin_lock+0x20/0x30 [1488061.829476] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.829487] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.829525] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.829527] [] ? del_timer_sync+0x52/0x60 [1488061.829563] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.829600] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.829639] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.829641] [] ? wake_up_state+0x20/0x20 [1488061.829679] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.829681] [] kthread+0xd1/0xe0 [1488061.829683] [] ? insert_kthread_work+0x40/0x40 [1488061.829685] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.829687] [] ? insert_kthread_work+0x40/0x40 [1488061.829707] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.848410] NMI watchdog: BUG: soft lockup - CPU#78 stuck for 22s! [ptlrpcd_03_10:54202] [1488061.848436] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.848444] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.848446] CPU: 78 PID: 54202 Comm: ptlrpcd_03_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.848447] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.848448] task: ffff8aaba9650000 ti: ffff8abafca54000 task.ti: ffff8abafca54000 [1488061.848451] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.848452] RSP: 0018:ffff8abafca57b78 EFLAGS: 00000246 [1488061.848453] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002710000 [1488061.848454] RDX: ffff8abaffadb780 RSI: 0000000000f10001 RDI: ffff8adafa7b1b00 [1488061.848455] RBP: ffff8abafca57b78 R08: ffff8abaffc5b780 R09: 0000000000000000 [1488061.848457] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab1562e5400 [1488061.848457] R13: 0005c4aa17ccdf60 R14: ffff8ab2fa371800 R15: 0000000000000007 [1488061.848459] FS: 00007ea1be070700(0000) GS:ffff8abaffc40000(0000) knlGS:0000000000000000 [1488061.848460] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.848461] CR2: 00007ec111ede00b CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.848462] Call Trace: [1488061.848464] [] queued_spin_lock_slowpath+0xb/0xf [1488061.848466] [] _raw_spin_lock+0x20/0x30 [1488061.848474] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.848485] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.848523] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.848525] [] ? del_timer_sync+0x52/0x60 [1488061.848562] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.848599] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.848638] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.848640] [] ? wake_up_state+0x20/0x20 [1488061.848679] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.848681] [] kthread+0xd1/0xe0 [1488061.848682] [] ? insert_kthread_work+0x40/0x40 [1488061.848684] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.848686] [] ? insert_kthread_work+0x40/0x40 [1488061.848706] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.857415] NMI watchdog: BUG: soft lockup - CPU#79 stuck for 22s! [ptlrpcd_07_00:54240] [1488061.857442] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.857449] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.857451] CPU: 79 PID: 54240 Comm: ptlrpcd_07_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.857452] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.857453] task: ffff8abafbc7b0c0 ti: ffff8abaf81e4000 task.ti: ffff8abaf81e4000 [1488061.857457] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.857457] RSP: 0018:ffff8abaf81e7b78 EFLAGS: 00000246 [1488061.857458] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002790000 [1488061.857459] RDX: ffff8a8aff29b780 RSI: 0000000000810001 RDI: ffff8adafa7b1b00 [1488061.857461] RBP: ffff8abaf81e7b78 R08: ffff8afb3fa5b780 R09: 0000000000000000 [1488061.857461] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aedf17a3200 [1488061.857462] R13: 0005c4aa17c00f40 R14: ffff8af7e55f5700 R15: 0000000000000007 [1488061.857464] FS: 00007f480aa7e880(0000) GS:ffff8afb3fa40000(0000) knlGS:0000000000000000 [1488061.857465] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.857466] CR2: 00007ecc283d9000 CR3: 0000006fdd120000 CR4: 00000000003407e0 [1488061.857467] Call Trace: [1488061.857469] [] queued_spin_lock_slowpath+0xb/0xf [1488061.857471] [] _raw_spin_lock+0x20/0x30 [1488061.857479] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.857490] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.857527] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.857565] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.857603] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.857641] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.857643] [] ? wake_up_state+0x20/0x20 [1488061.857682] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.857684] [] kthread+0xd1/0xe0 [1488061.857686] [] ? insert_kthread_work+0x40/0x40 [1488061.857687] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.857689] [] ? insert_kthread_work+0x40/0x40 [1488061.857709] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.865415] NMI watchdog: BUG: soft lockup - CPU#80 stuck for 22s! [ptlrpcd_00_01:54157] [1488061.865441] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.865448] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.865451] CPU: 80 PID: 54157 Comm: ptlrpcd_00_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.865452] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.865453] task: ffff8afaf5f32080 ti: ffff8abaed21c000 task.ti: ffff8abaed21c000 [1488061.865456] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.865457] RSP: 0018:ffff8abaed21fb78 EFLAGS: 00000246 [1488061.865457] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002810000 [1488061.865459] RDX: ffff8abaffcdb780 RSI: 0000000002f10001 RDI: ffff8adafa7b1b00 [1488061.865459] RBP: ffff8abaed21fb78 R08: ffff8a8aff49b780 R09: 0000000000000000 [1488061.865460] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c6a00 [1488061.865462] R13: 0005c4aa187a05c0 R14: ffff8a90d9b10300 R15: 0000000000000007 [1488061.865463] FS: 00007ea1b505e700(0000) GS:ffff8a8aff480000(0000) knlGS:0000000000000000 [1488061.865464] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.865465] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.865466] Call Trace: [1488061.865468] [] queued_spin_lock_slowpath+0xb/0xf [1488061.865470] [] _raw_spin_lock+0x20/0x30 [1488061.865478] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.865489] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.865527] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.865564] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.865601] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.865639] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.865641] [] ? wake_up_state+0x20/0x20 [1488061.865679] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.865681] [] kthread+0xd1/0xe0 [1488061.865683] [] ? insert_kthread_work+0x40/0x40 [1488061.865685] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.865687] [] ? insert_kthread_work+0x40/0x40 [1488061.865707] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.874417] NMI watchdog: BUG: soft lockup - CPU#81 stuck for 22s! [ptlrpcd_04_00:54204] [1488061.874444] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.874451] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.874454] CPU: 81 PID: 54204 Comm: ptlrpcd_04_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.874455] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.874456] task: ffff8aaba9652080 ti: ffff8abaf1608000 task.ti: ffff8abaf1608000 [1488061.874459] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.874460] RSP: 0018:ffff8abaf160bb78 EFLAGS: 00000246 [1488061.874461] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002890000 [1488061.874462] RDX: ffff8afb3f99b780 RSI: 0000000001b90001 RDI: ffff8adafa7b1b00 [1488061.874463] RBP: ffff8abaf160bb78 R08: ffff8acaffc9b780 R09: 0000000000000000 [1488061.874464] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac88f138200 [1488061.874465] R13: 0005c4aa17b5f960 R14: ffff8ac182749500 R15: 0000000000000007 [1488061.874467] FS: 00007ea1dd8af700(0000) GS:ffff8acaffc80000(0000) knlGS:0000000000000000 [1488061.874467] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.874469] CR2: 00007ef77c795f00 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.874469] Call Trace: [1488061.874471] [] queued_spin_lock_slowpath+0xb/0xf [1488061.874473] [] _raw_spin_lock+0x20/0x30 [1488061.874481] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.874492] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.874530] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.874568] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.874605] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.874644] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.874646] [] ? wake_up_state+0x20/0x20 [1488061.874684] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.874686] [] kthread+0xd1/0xe0 [1488061.874688] [] ? insert_kthread_work+0x40/0x40 [1488061.874690] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.874692] [] ? insert_kthread_work+0x40/0x40 [1488061.874712] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.884415] NMI watchdog: BUG: soft lockup - CPU#82 stuck for 22s! [ptlrpcd_01_03:54171] [1488061.884444] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.884451] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.884454] CPU: 82 PID: 54171 Comm: ptlrpcd_01_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.884455] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.884456] task: ffff8abaf6344100 ti: ffff8abafcd88000 task.ti: ffff8abafcd88000 [1488061.884459] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.884460] RSP: 0018:ffff8abafcd8bb78 EFLAGS: 00000246 [1488061.884461] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002910000 [1488061.884462] RDX: ffff8adaffa5b780 RSI: 0000000000590001 RDI: ffff8adafa7b1b00 [1488061.884463] RBP: ffff8abafcd8bb78 R08: ffff8a9affc9b780 R09: 0000000000000000 [1488061.884464] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8cf23c7c00 [1488061.884465] R13: 0005c4aa17a41660 R14: ffff8a97cdd14500 R15: 0000000000000007 [1488061.884467] FS: 00007ea1c9887700(0000) GS:ffff8a9affc80000(0000) knlGS:0000000000000000 [1488061.884467] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.884468] CR2: 00007f8c2c495000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.884469] Call Trace: [1488061.884472] [] queued_spin_lock_slowpath+0xb/0xf [1488061.884473] [] _raw_spin_lock+0x20/0x30 [1488061.884481] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.884492] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.884530] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.884568] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.884605] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.884643] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.884645] [] ? wake_up_state+0x20/0x20 [1488061.884684] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.884686] [] kthread+0xd1/0xe0 [1488061.884688] [] ? insert_kthread_work+0x40/0x40 [1488061.884689] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.884691] [] ? insert_kthread_work+0x40/0x40 [1488061.884712] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.911419] NMI watchdog: BUG: soft lockup - CPU#85 stuck for 22s! [ptlrpcd_06_07:54235] [1488061.911447] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.911454] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.911457] CPU: 85 PID: 54235 Comm: ptlrpcd_06_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.911458] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.911459] task: ffff8abaf4785140 ti: ffff8abafc788000 task.ti: ffff8abafc788000 [1488061.911462] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.911463] RSP: 0018:ffff8abafc78bb78 EFLAGS: 00000246 [1488061.911464] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002a90000 [1488061.911465] RDX: ffff8aeaffa9b780 RSI: 0000000000a90001 RDI: ffff8adafa7b1b00 [1488061.911466] RBP: ffff8abafc78bb78 R08: ffff8aeaffc9b780 R09: 0000000000000000 [1488061.911467] R10: 0000000000000000 R11: 000000000000000f R12: ffff8add4abcd800 [1488061.911468] R13: 0005c4aa17cfee70 R14: ffff8ae7b12db600 R15: 0000000000000007 [1488061.911469] FS: 00007ea1dc8ad700(0000) GS:ffff8aeaffc80000(0000) knlGS:0000000000000000 [1488061.911470] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.911471] CR2: 00007ec44bcee000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.911472] Call Trace: [1488061.911474] [] queued_spin_lock_slowpath+0xb/0xf [1488061.911476] [] _raw_spin_lock+0x20/0x30 [1488061.911484] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.911495] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.911533] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.911571] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.911608] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.911646] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.911648] [] ? wake_up_state+0x20/0x20 [1488061.911687] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.911689] [] kthread+0xd1/0xe0 [1488061.911691] [] ? insert_kthread_work+0x40/0x40 [1488061.911693] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.911694] [] ? insert_kthread_work+0x40/0x40 [1488061.911715] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.921416] NMI watchdog: BUG: soft lockup - CPU#86 stuck for 22s! [ptlrpcd_03_05:54197] [1488061.921444] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.921452] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.921454] CPU: 86 PID: 54197 Comm: ptlrpcd_03_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.921456] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.921457] task: ffff8abaf6392080 ti: ffff8abafab64000 task.ti: ffff8abafab64000 [1488061.921460] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.921461] RSP: 0018:ffff8abafab67b78 EFLAGS: 00000246 [1488061.921462] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002b10000 [1488061.921463] RDX: ffff8adaffcdb780 RSI: 0000000002d90001 RDI: ffff8adafa7b1b00 [1488061.921464] RBP: ffff8abafab67b78 R08: ffff8abaffc9b780 R09: 0000000000000000 [1488061.921465] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aabdc30be00 [1488061.921466] R13: 0005c4aa17d36450 R14: ffff8ab441784b00 R15: 0000000000000007 [1488061.921468] FS: 00007ea1b385b700(0000) GS:ffff8abaffc80000(0000) knlGS:0000000000000000 [1488061.921469] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.921470] CR2: 00007edf3fa26dc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.921471] Call Trace: [1488061.921473] [] queued_spin_lock_slowpath+0xb/0xf [1488061.921475] [] _raw_spin_lock+0x20/0x30 [1488061.921483] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.921493] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.921531] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.921568] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.921605] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.921644] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.921646] [] ? wake_up_state+0x20/0x20 [1488061.921684] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.921686] [] kthread+0xd1/0xe0 [1488061.921688] [] ? insert_kthread_work+0x40/0x40 [1488061.921690] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.921692] [] ? insert_kthread_work+0x40/0x40 [1488061.921711] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.930421] NMI watchdog: BUG: soft lockup - CPU#87 stuck for 22s! [ptlrpcd_07_01:54241] [1488061.930448] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.930456] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.930459] CPU: 87 PID: 54241 Comm: ptlrpcd_07_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.930459] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.930461] task: ffff8abafbc7c100 ti: ffff8abaed65c000 task.ti: ffff8abaed65c000 [1488061.930464] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.930464] RSP: 0018:ffff8abaed65fb78 EFLAGS: 00000246 [1488061.930465] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002b90000 [1488061.930466] RDX: ffff8afb3f91b780 RSI: 0000000001390001 RDI: ffff8adafa7b1b00 [1488061.930468] RBP: ffff8abaed65fb78 R08: ffff8afb3fa9b780 R09: 0000000000000000 [1488061.930469] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af635444000 [1488061.930470] R13: 0005c4aa17cd8bb0 R14: ffff8af616504800 R15: 0000000000000007 [1488061.930471] FS: 00007ea1d2098700(0000) GS:ffff8afb3fa80000(0000) knlGS:0000000000000000 [1488061.930472] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.930473] CR2: 00007eec1a4e7dc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.930474] Call Trace: [1488061.930476] [] queued_spin_lock_slowpath+0xb/0xf [1488061.930478] [] _raw_spin_lock+0x20/0x30 [1488061.930486] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.930497] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.930535] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.930576] [] ? lustre_msg_buf_v2+0x6e/0x1b0 [ptlrpc] [1488061.930614] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.930651] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.930690] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.930692] [] ? wake_up_state+0x20/0x20 [1488061.930731] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.930733] [] kthread+0xd1/0xe0 [1488061.930734] [] ? insert_kthread_work+0x40/0x40 [1488061.930736] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.930738] [] ? insert_kthread_work+0x40/0x40 [1488061.930758] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.938418] NMI watchdog: BUG: soft lockup - CPU#88 stuck for 22s! [ptlrpcd_00_00:54156] [1488061.938446] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.938453] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.938456] CPU: 88 PID: 54156 Comm: ptlrpcd_00_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.938457] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.938458] task: ffff8afaf5f30000 ti: ffff8abaf9e98000 task.ti: ffff8abaf9e98000 [1488061.938461] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.938462] RSP: 0018:ffff8abaf9e9bb78 EFLAGS: 00000246 [1488061.938463] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002c10000 [1488061.938464] RDX: ffff8aaaffadb780 RSI: 0000000000e10001 RDI: ffff8adafa7b1b00 [1488061.938465] RBP: ffff8abaf9e9bb78 R08: ffff8a8aff4db780 R09: 0000000000000000 [1488061.938466] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa198af4a00 [1488061.938467] R13: 0005c4aa187a16a0 R14: ffff8a97c5ae2a00 R15: 0000000000000007 [1488061.938469] FS: 00007fac50325880(0000) GS:ffff8a8aff4c0000(0000) knlGS:0000000000000000 [1488061.938469] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.938471] CR2: 00007fbad3d81000 CR3: 0000007a97470000 CR4: 00000000003407e0 [1488061.938472] Call Trace: [1488061.938474] [] queued_spin_lock_slowpath+0xb/0xf [1488061.938476] [] _raw_spin_lock+0x20/0x30 [1488061.938484] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.938495] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.938532] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.938569] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.938606] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.938645] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.938647] [] ? wake_up_state+0x20/0x20 [1488061.938685] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.938687] [] kthread+0xd1/0xe0 [1488061.938689] [] ? insert_kthread_work+0x40/0x40 [1488061.938691] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.938692] [] ? insert_kthread_work+0x40/0x40 [1488061.938712] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.947419] NMI watchdog: BUG: soft lockup - CPU#89 stuck for 22s! [ptlrpcd_04_03:54207] [1488061.947447] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.947454] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.947457] CPU: 89 PID: 54207 Comm: ptlrpcd_04_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.947458] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.947459] task: ffff8aaba9655140 ti: ffff8abaed254000 task.ti: ffff8abaed254000 [1488061.947462] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488061.947463] RSP: 0018:ffff8abaed257b78 EFLAGS: 00000246 [1488061.947464] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002c90000 [1488061.947465] RDX: ffff8adaffb9b780 RSI: 0000000001990001 RDI: ffff8adafa7b1b00 [1488061.947466] RBP: ffff8abaed257b78 R08: ffff8acaffcdb780 R09: 0000000000000000 [1488061.947467] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac1a1ad2400 [1488061.947468] R13: 0005c4aa17cc9980 R14: ffff8ac1a1b0bf00 R15: 0000000000000007 [1488061.947470] FS: 00007ea1df8b3700(0000) GS:ffff8acaffcc0000(0000) knlGS:0000000000000000 [1488061.947470] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.947472] CR2: 00007ee0c37e002a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.947472] Call Trace: [1488061.947474] [] queued_spin_lock_slowpath+0xb/0xf [1488061.947477] [] _raw_spin_lock+0x20/0x30 [1488061.947485] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.947495] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.947532] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.947570] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.947607] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.947645] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.947647] [] ? wake_up_state+0x20/0x20 [1488061.947685] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.947687] [] kthread+0xd1/0xe0 [1488061.947689] [] ? insert_kthread_work+0x40/0x40 [1488061.947691] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.947693] [] ? insert_kthread_work+0x40/0x40 [1488061.947712] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488061.974417] NMI watchdog: BUG: soft lockup - CPU#92 stuck for 22s! [ptlrpcd_02_01:54181] [1488061.974444] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.974451] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.974454] CPU: 92 PID: 54181 Comm: ptlrpcd_02_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.974455] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.974456] task: ffff8abaee640000 ti: ffff8abaf8110000 task.ti: ffff8abaf8110000 [1488061.974459] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.974460] RSP: 0018:ffff8abaf8113b78 EFLAGS: 00000246 [1488061.974461] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002e10000 [1488061.974462] RDX: ffff8acaffadb780 RSI: 0000000000c90001 RDI: ffff8adafa7b1b00 [1488061.974463] RBP: ffff8abaf8113b78 R08: ffff8aaaffcdb780 R09: 0000000000000000 [1488061.974464] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa9efa17c00 [1488061.974465] R13: 0005c4aa17cf9150 R14: ffff8aa699edf500 R15: 0000000000000007 [1488061.974466] FS: 00007ea1dd0ae700(0000) GS:ffff8aaaffcc0000(0000) knlGS:0000000000000000 [1488061.974467] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.974468] CR2: 00007f8c1f871000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.974469] Call Trace: [1488061.974471] [] queued_spin_lock_slowpath+0xb/0xf [1488061.974473] [] _raw_spin_lock+0x20/0x30 [1488061.974481] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.974492] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.974529] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.974567] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.974603] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.974641] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.974644] [] ? wake_up_state+0x20/0x20 [1488061.974682] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.974684] [] kthread+0xd1/0xe0 [1488061.974686] [] ? insert_kthread_work+0x40/0x40 [1488061.974687] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.974690] [] ? insert_kthread_work+0x40/0x40 [1488061.974709] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488061.983421] NMI watchdog: BUG: soft lockup - CPU#93 stuck for 22s! [ptlrpcd_06_03:54231] [1488061.983449] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488061.983456] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488061.983459] CPU: 93 PID: 54231 Comm: ptlrpcd_06_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488061.983460] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488061.983461] task: ffff8abaf4781040 ti: ffff8abaebfe4000 task.ti: ffff8abaebfe4000 [1488061.983464] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488061.983465] RSP: 0018:ffff8abaebfe7b78 EFLAGS: 00000246 [1488061.983466] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002e90000 [1488061.983467] RDX: ffff8acaffa5b780 RSI: 0000000000490001 RDI: ffff8adafa7b1b00 [1488061.983468] RBP: ffff8abaebfe7b78 R08: ffff8aeaffcdb780 R09: 0000000000000000 [1488061.983469] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adffbec7600 [1488061.983470] R13: 0005c4aa17d15190 R14: ffff8ae87a880f00 R15: 0000000000000007 [1488061.983472] FS: 00007ea1b485d700(0000) GS:ffff8aeaffcc0000(0000) knlGS:0000000000000000 [1488061.983473] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488061.983474] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488061.983475] Call Trace: [1488061.983477] [] queued_spin_lock_slowpath+0xb/0xf [1488061.983479] [] _raw_spin_lock+0x20/0x30 [1488061.983487] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488061.983498] [] LNetMDUnlink+0x78/0x180 [lnet] [1488061.983536] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488061.983574] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488061.983611] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488061.983650] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488061.983652] [] ? wake_up_state+0x20/0x20 [1488061.983691] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488061.983693] [] kthread+0xd1/0xe0 [1488061.983694] [] ? insert_kthread_work+0x40/0x40 [1488061.983696] [] ret_from_fork_nospec_begin+0xe/0x21 [1488061.983698] [] ? insert_kthread_work+0x40/0x40 [1488061.983718] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488062.002421] NMI watchdog: BUG: soft lockup - CPU#95 stuck for 22s! [ptlrpcd_07_04:54244] [1488062.002447] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488062.002454] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488062.002457] CPU: 95 PID: 54244 Comm: ptlrpcd_07_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488062.002458] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488062.002459] task: ffff8abafc520000 ti: ffff8abaf8a88000 task.ti: ffff8abaf8a88000 [1488062.002462] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488062.002463] RSP: 0018:ffff8abaf8a8bb78 EFLAGS: 00000246 [1488062.002464] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002f90000 [1488062.002465] RDX: ffff8aaaffc1b780 RSI: 0000000002210001 RDI: ffff8adafa7b1b00 [1488062.002466] RBP: ffff8abaf8a8bb78 R08: ffff8afb3fadb780 R09: 0000000000000000 [1488062.002467] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aee00395600 [1488062.002468] R13: 0005c4aa17c03340 R14: ffff8afaa4fd9b00 R15: 0000000000000007 [1488062.002470] FS: 00007f7e5fc01700(0000) GS:ffff8afb3fac0000(0000) knlGS:0000000000000000 [1488062.002470] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488062.002472] CR2: 00007ecbd22c6000 CR3: 0000006035e02000 CR4: 00000000003407e0 [1488062.002472] Call Trace: [1488062.002474] [] queued_spin_lock_slowpath+0xb/0xf [1488062.002477] [] _raw_spin_lock+0x20/0x30 [1488062.002485] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488062.002495] [] LNetMDUnlink+0x78/0x180 [lnet] [1488062.002533] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488062.002571] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488062.002608] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488062.002647] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488062.002649] [] ? wake_up_state+0x20/0x20 [1488062.002688] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488062.002689] [] kthread+0xd1/0xe0 [1488062.002691] [] ? insert_kthread_work+0x40/0x40 [1488062.002693] [] ret_from_fork_nospec_begin+0xe/0x21 [1488062.002695] [] ? insert_kthread_work+0x40/0x40 [1488062.002715] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488065.079624] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ptlrpcd_00_08:54164] [1488065.079652] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.079660] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.079663] CPU: 0 PID: 54164 Comm: ptlrpcd_00_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.079663] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.079665] task: ffff8abaf815c100 ti: ffff8abaecad0000 task.ti: ffff8abaecad0000 [1488065.079668] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488065.079669] RSP: 0018:ffff8abaecad3b78 EFLAGS: 00000246 [1488065.079670] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000010000 [1488065.079671] RDX: ffff8a9affc5b780 RSI: 0000000002510001 RDI: ffff8adafa7b1b00 [1488065.079672] RBP: ffff8abaecad3b78 R08: ffff8a8aff21b780 R09: 0000000000000000 [1488065.079673] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9294bafe00 [1488065.079674] R13: 0005c4aa186d9210 R14: ffff8af647b88c00 R15: 0000000000000007 [1488065.079676] FS: 00007ea1dd0ae700(0000) GS:ffff8a8aff200000(0000) knlGS:0000000000000000 [1488065.079677] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.079678] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407f0 [1488065.079679] Call Trace: [1488065.079681] [] queued_spin_lock_slowpath+0xb/0xf [1488065.079683] [] _raw_spin_lock+0x20/0x30 [1488065.079691] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.079702] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.079740] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.079777] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.079814] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.079852] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.079855] [] ? wake_up_state+0x20/0x20 [1488065.079893] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.079895] [] kthread+0xd1/0xe0 [1488065.079897] [] ? insert_kthread_work+0x40/0x40 [1488065.079899] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.079901] [] ? insert_kthread_work+0x40/0x40 [1488065.079921] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488065.309640] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ptlrpcd_05_02:54218] [1488065.309668] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.309675] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.309677] CPU: 19 PID: 54218 Comm: ptlrpcd_05_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.309678] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.309679] task: ffff8abaee63a080 ti: ffff8abaecb24000 task.ti: ffff8abaecb24000 [1488065.309682] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488065.309683] RSP: 0018:ffff8abaecb27b78 EFLAGS: 00000246 [1488065.309684] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000990000 [1488065.309684] RDX: ffff8aaaffc9b780 RSI: 0000000002a10001 RDI: ffff8adafa7b1b00 [1488065.309685] RBP: ffff8abaecb27b78 R08: ffff8adaffa9b780 R09: 0000000000000000 [1488065.309686] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad0543e2a00 [1488065.309686] R13: 0005c4aa1786d7b0 R14: ffff8adab3c4b900 R15: 0000000000000007 [1488065.309688] FS: 00007ea1bc06c700(0000) GS:ffff8adaffa80000(0000) knlGS:0000000000000000 [1488065.309689] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.309689] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.309690] Call Trace: [1488065.309692] [] queued_spin_lock_slowpath+0xb/0xf [1488065.309694] [] _raw_spin_lock+0x20/0x30 [1488065.309702] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.309712] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.309750] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.309752] [] ? del_timer_sync+0x52/0x60 [1488065.309789] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.309826] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.309864] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.309867] [] ? wake_up_state+0x20/0x20 [1488065.309905] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.309907] [] kthread+0xd1/0xe0 [1488065.309909] [] ? insert_kthread_work+0x40/0x40 [1488065.309910] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.309912] [] ? insert_kthread_work+0x40/0x40 [1488065.309932] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488065.355639] NMI watchdog: BUG: soft lockup - CPU#24 stuck for 22s! [ptlrpcd_00_11:54167] [1488065.355667] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.355675] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.355677] CPU: 24 PID: 54167 Comm: ptlrpcd_00_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.355678] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.355679] task: ffff8abaf6340000 ti: ffff8abafcf38000 task.ti: ffff8abafcf38000 [1488065.355682] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488065.355683] RSP: 0018:ffff8abafcf3bb78 EFLAGS: 00000246 [1488065.355685] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000c10000 [1488065.355685] RDX: ffff8a9affcdb780 RSI: 0000000002d10001 RDI: ffff8adafa7b1b00 [1488065.355686] RBP: ffff8abafcf3bb78 R08: ffff8a8aff2db780 R09: 0000000000000000 [1488065.355688] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c7c00 [1488065.355688] R13: 0005c4aa187a0800 R14: ffff8a9a09e6d400 R15: 0000000000000007 [1488065.355690] FS: 00007ea1b9867700(0000) GS:ffff8a8aff2c0000(0000) knlGS:0000000000000000 [1488065.355691] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.355692] CR2: 00007f03faf61003 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.355693] Call Trace: [1488065.355695] [] queued_spin_lock_slowpath+0xb/0xf [1488065.355697] [] _raw_spin_lock+0x20/0x30 [1488065.355705] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.355716] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.355754] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.355756] [] ? del_timer_sync+0x52/0x60 [1488065.355794] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.355831] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.355869] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.355872] [] ? wake_up_state+0x20/0x20 [1488065.355910] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.355912] [] kthread+0xd1/0xe0 [1488065.355914] [] ? insert_kthread_work+0x40/0x40 [1488065.355915] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.355917] [] ? insert_kthread_work+0x40/0x40 [1488065.355937] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488065.455649] NMI watchdog: BUG: soft lockup - CPU#35 stuck for 22s! [ptlrpcd_05_09:54225] [1488065.455674] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.455681] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.455684] CPU: 35 PID: 54225 Comm: ptlrpcd_05_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.455685] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.455685] task: ffff8abafb33a080 ti: ffff8abafc488000 task.ti: ffff8abafc488000 [1488065.455688] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488065.455689] RSP: 0018:ffff8abafc48bb78 EFLAGS: 00000246 [1488065.455690] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001190000 [1488065.455690] RDX: ffff8a8aff41b780 RSI: 0000000002010001 RDI: ffff8adafa7b1b00 [1488065.455691] RBP: ffff8abafc48bb78 R08: ffff8adaffb1b780 R09: 0000000000000000 [1488065.455692] R10: 0000000000000000 R11: 000000000000000f R12: ffff8accfe737200 [1488065.455693] R13: 0005c4aa17a87530 R14: ffff8ad2bf7d4800 R15: 0000000000000007 [1488065.455694] FS: 00007ea1d489d700(0000) GS:ffff8adaffb00000(0000) knlGS:0000000000000000 [1488065.455695] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.455695] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.455696] Call Trace: [1488065.455698] [] queued_spin_lock_slowpath+0xb/0xf [1488065.455700] [] _raw_spin_lock+0x20/0x30 [1488065.455708] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.455718] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.455756] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.455758] [] ? del_timer_sync+0x52/0x60 [1488065.455795] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.455831] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.455870] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.455872] [] ? wake_up_state+0x20/0x20 [1488065.455910] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.455912] [] kthread+0xd1/0xe0 [1488065.455913] [] ? insert_kthread_work+0x40/0x40 [1488065.455915] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.455917] [] ? insert_kthread_work+0x40/0x40 [1488065.455936] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488065.575654] NMI watchdog: BUG: soft lockup - CPU#48 stuck for 23s! [ptlrpcd_00_05:54161] [1488065.575682] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.575689] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.575692] CPU: 48 PID: 54161 Comm: ptlrpcd_00_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.575692] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.575694] task: ffff8abaf8159040 ti: ffff8abafc6a8000 task.ti: ffff8abafc6a8000 [1488065.575697] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488065.575698] RSP: 0018:ffff8abafc6abb78 EFLAGS: 00000246 [1488065.575698] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001810000 [1488065.575700] RDX: ffff8adaffbdb780 RSI: 0000000001d90001 RDI: ffff8adafa7b1b00 [1488065.575700] RBP: ffff8abafc6abb78 R08: ffff8a8aff39b780 R09: 0000000000000000 [1488065.575701] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c0400 [1488065.575703] R13: 0005c4aa1879fad0 R14: ffff8a7b02ec1800 R15: 0000000000000007 [1488065.575704] FS: 00007ea1af052700(0000) GS:ffff8a8aff380000(0000) knlGS:0000000000000000 [1488065.575705] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.575706] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.575707] Call Trace: [1488065.575710] [] queued_spin_lock_slowpath+0xb/0xf [1488065.575712] [] _raw_spin_lock+0x20/0x30 [1488065.575720] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.575731] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.575769] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.575806] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.575843] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.575882] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.575884] [] ? wake_up_state+0x20/0x20 [1488065.575923] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.575925] [] kthread+0xd1/0xe0 [1488065.575927] [] ? insert_kthread_work+0x40/0x40 [1488065.575929] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.575931] [] ? insert_kthread_work+0x40/0x40 [1488065.575952] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488065.647659] NMI watchdog: BUG: soft lockup - CPU#56 stuck for 23s! [ptlrpcd_00_03:54159] [1488065.647685] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.647693] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.647695] CPU: 56 PID: 54159 Comm: ptlrpcd_00_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.647696] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.647697] task: ffff8afaf5f36180 ti: ffff8abafb264000 task.ti: ffff8abafb264000 [1488065.647700] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488065.647701] RSP: 0018:ffff8abafb267b78 EFLAGS: 00000246 [1488065.647703] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001c10000 [1488065.647703] RDX: ffff8abaffb9b780 RSI: 0000000001b10001 RDI: ffff8adafa7b1b00 [1488065.647704] RBP: ffff8abafb267b78 R08: ffff8a8aff3db780 R09: 0000000000000000 [1488065.647705] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c2c00 [1488065.647707] R13: 0005c4aa1879ff30 R14: ffff8a90d9b12d00 R15: 0000000000000007 [1488065.647708] FS: 00007fc0d2114880(0000) GS:ffff8a8aff3c0000(0000) knlGS:0000000000000000 [1488065.647709] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.647710] CR2: 00007f012f4f9005 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.647711] Call Trace: [1488065.647713] [] queued_spin_lock_slowpath+0xb/0xf [1488065.647715] [] _raw_spin_lock+0x20/0x30 [1488065.647723] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.647734] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.647772] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.647810] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.647847] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.647886] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.647888] [] ? wake_up_state+0x20/0x20 [1488065.647926] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.647928] [] kthread+0xd1/0xe0 [1488065.647930] [] ? insert_kthread_work+0x40/0x40 [1488065.647932] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.647934] [] ? insert_kthread_work+0x40/0x40 [1488065.647954] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488065.692662] NMI watchdog: BUG: soft lockup - CPU#61 stuck for 23s! [ptlrpcd_06_08:54236] [1488065.692688] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.692696] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.692698] CPU: 61 PID: 54236 Comm: ptlrpcd_06_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.692699] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.692701] task: ffff8abaf4786180 ti: ffff8abaeb6a8000 task.ti: ffff8abaeb6a8000 [1488065.692704] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488065.692705] RSP: 0018:ffff8abaeb6abb78 EFLAGS: 00000246 [1488065.692706] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001e90000 [1488065.692707] RDX: ffff8a9affbdb780 RSI: 0000000001d10001 RDI: ffff8adafa7b1b00 [1488065.692708] RBP: ffff8abaeb6abb78 R08: ffff8aeaffbdb780 R09: 0000000000000000 [1488065.692709] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adeb78e0200 [1488065.692710] R13: 0005c4aa17c27490 R14: ffff8ab98a57ec00 R15: 0000000000000007 [1488065.692712] FS: 00007ea1b585f700(0000) GS:ffff8aeaffbc0000(0000) knlGS:0000000000000000 [1488065.692713] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.692714] CR2: 00007edf22602002 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.692715] Call Trace: [1488065.692717] [] queued_spin_lock_slowpath+0xb/0xf [1488065.692719] [] _raw_spin_lock+0x20/0x30 [1488065.692727] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.692737] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.692776] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.692778] [] ? del_timer_sync+0x52/0x60 [1488065.692815] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.692853] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.692891] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.692893] [] ? wake_up_state+0x20/0x20 [1488065.692931] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.692934] [] kthread+0xd1/0xe0 [1488065.692935] [] ? insert_kthread_work+0x40/0x40 [1488065.692937] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.692939] [] ? insert_kthread_work+0x40/0x40 [1488065.692959] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488065.748670] NMI watchdog: BUG: soft lockup - CPU#67 stuck for 23s! [ptlrpcd_05_07:54223] [1488065.748697] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488065.748704] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488065.748706] CPU: 67 PID: 54223 Comm: ptlrpcd_05_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488065.748707] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488065.748708] task: ffff8abafb338000 ti: ffff8abafc59c000 task.ti: ffff8abafc59c000 [1488065.748710] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488065.748711] RSP: 0018:ffff8abafc59fb78 EFLAGS: 00000246 [1488065.748712] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002190000 [1488065.748713] RDX: ffff8a8aff31b780 RSI: 0000000001010001 RDI: ffff8adafa7b1b00 [1488065.748713] RBP: ffff8abafc59fb78 R08: ffff8adaffc1b780 R09: 0000000000000000 [1488065.748714] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad080454600 [1488065.748715] R13: 0005c4aa17af5920 R14: ffff8ad4ecdef800 R15: 0000000000000007 [1488065.748716] FS: 00007ea1da8a9700(0000) GS:ffff8adaffc00000(0000) knlGS:0000000000000000 [1488065.748717] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488065.748717] CR2: 00007f3af9191ac0 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488065.748718] Call Trace: [1488065.748720] [] queued_spin_lock_slowpath+0xb/0xf [1488065.748721] [] _raw_spin_lock+0x20/0x30 [1488065.748729] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488065.748739] [] LNetMDUnlink+0x78/0x180 [lnet] [1488065.748777] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488065.748815] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488065.748852] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488065.748890] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488065.748892] [] ? wake_up_state+0x20/0x20 [1488065.748930] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488065.748931] [] kthread+0xd1/0xe0 [1488065.748933] [] ? insert_kthread_work+0x40/0x40 [1488065.748935] [] ret_from_fork_nospec_begin+0xe/0x21 [1488065.748936] [] ? insert_kthread_work+0x40/0x40 [1488065.748956] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488069.812932] NMI watchdog: BUG: soft lockup - CPU#74 stuck for 23s! [ptlrpcd_01_08:54176] [1488069.812959] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488069.812967] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488069.812970] CPU: 74 PID: 54176 Comm: ptlrpcd_01_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488069.812970] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488069.812971] task: ffff8abaf4daa080 ti: ffff8abaf63f0000 task.ti: ffff8abaf63f0000 [1488069.812975] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488069.812976] RSP: 0018:ffff8abaf63f3b78 EFLAGS: 00000246 [1488069.812977] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002510000 [1488069.812978] RDX: ffff8aaaffb5b780 RSI: 0000000001610001 RDI: ffff8adafa7b1b00 [1488069.812978] RBP: ffff8abaf63f3b78 R08: ffff8a9affc5b780 R09: 0000000000000000 [1488069.812980] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a95b8e0f400 [1488069.812981] R13: 0005c4aa179d8f00 R14: ffff8a8fdf5eb300 R15: 0000000000000007 [1488069.812982] FS: 00007ea1d409c700(0000) GS:ffff8a9affc40000(0000) knlGS:0000000000000000 [1488069.812983] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488069.812984] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488069.812985] Call Trace: [1488069.812987] [] queued_spin_lock_slowpath+0xb/0xf [1488069.812989] [] _raw_spin_lock+0x20/0x30 [1488069.812997] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488069.813008] [] LNetMDUnlink+0x78/0x180 [lnet] [1488069.813046] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488069.813048] [] ? del_timer_sync+0x52/0x60 [1488069.813085] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488069.813122] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488069.813160] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488069.813162] [] ? wake_up_state+0x20/0x20 [1488069.813200] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488069.813202] [] kthread+0xd1/0xe0 [1488069.813204] [] ? insert_kthread_work+0x40/0x40 [1488069.813206] [] ret_from_fork_nospec_begin+0xe/0x21 [1488069.813208] [] ? insert_kthread_work+0x40/0x40 [1488069.813228] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488069.893937] NMI watchdog: BUG: soft lockup - CPU#83 stuck for 23s! [ptlrpcd_05_05:54221] [1488069.893963] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488069.893970] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488069.893972] CPU: 83 PID: 54221 Comm: ptlrpcd_05_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488069.893973] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488069.893973] task: ffff8abaee63d140 ti: ffff8abaf87c8000 task.ti: ffff8abaf87c8000 [1488069.893976] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488069.893977] RSP: 0018:ffff8abaf87cbb78 EFLAGS: 00000246 [1488069.893978] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002990000 [1488069.893978] RDX: ffff8abaffa5b780 RSI: 0000000000710001 RDI: ffff8adafa7b1b00 [1488069.893979] RBP: ffff8abaf87cbb78 R08: ffff8adaffc9b780 R09: 0000000000000000 [1488069.893980] R10: 0000000000000000 R11: 000000000000000f R12: ffff8acba1d9a000 [1488069.893981] R13: 0005c4aa17ad97e0 R14: ffff8ad6e02a9500 R15: 0000000000000007 [1488069.893982] FS: 00007ea1cc08c700(0000) GS:ffff8adaffc80000(0000) knlGS:0000000000000000 [1488069.893983] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488069.893983] CR2: 00007f8c121f1000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488069.893984] Call Trace: [1488069.893986] [] queued_spin_lock_slowpath+0xb/0xf [1488069.893987] [] _raw_spin_lock+0x20/0x30 [1488069.893995] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488069.894005] [] LNetMDUnlink+0x78/0x180 [lnet] [1488069.894043] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488069.894080] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488069.894116] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488069.894154] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488069.894156] [] ? wake_up_state+0x20/0x20 [1488069.894194] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488069.894195] [] kthread+0xd1/0xe0 [1488069.894197] [] ? insert_kthread_work+0x40/0x40 [1488069.894199] [] ret_from_fork_nospec_begin+0xe/0x21 [1488069.894200] [] ? insert_kthread_work+0x40/0x40 [1488069.894219] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488069.957946] NMI watchdog: BUG: soft lockup - CPU#90 stuck for 23s! [ptlrpcd_01_06:54174] [1488069.957973] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488069.957981] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488069.957984] CPU: 90 PID: 54174 Comm: ptlrpcd_01_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488069.957985] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488069.957986] task: ffff8abaf4da8000 ti: ffff8abaf8f0c000 task.ti: ffff8abaf8f0c000 [1488069.957989] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488069.957990] RSP: 0018:ffff8abaf8f0fb78 EFLAGS: 00000246 [1488069.957991] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002d10000 [1488069.957992] RDX: ffff8a8aff35b780 RSI: 0000000001410001 RDI: ffff8adafa7b1b00 [1488069.957993] RBP: ffff8abaf8f0fb78 R08: ffff8a9affcdb780 R09: 0000000000000000 [1488069.957994] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9800c84200 [1488069.957995] R13: 0005c4aa179e6300 R14: ffff8a8ffc87d100 R15: 0000000000000007 [1488069.957997] FS: 00007ea1b9867700(0000) GS:ffff8a9affcc0000(0000) knlGS:0000000000000000 [1488069.957998] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488069.957999] CR2: 00007fb3d30fe000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488069.958000] Call Trace: [1488069.958002] [] queued_spin_lock_slowpath+0xb/0xf [1488069.958004] [] _raw_spin_lock+0x20/0x30 [1488069.958012] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488069.958023] [] LNetMDUnlink+0x78/0x180 [lnet] [1488069.958061] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488069.958099] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488069.958135] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488069.958174] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488069.958176] [] ? wake_up_state+0x20/0x20 [1488069.958215] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488069.958217] [] kthread+0xd1/0xe0 [1488069.958219] [] ? insert_kthread_work+0x40/0x40 [1488069.958221] [] ret_from_fork_nospec_begin+0xe/0x21 [1488069.958223] [] ? insert_kthread_work+0x40/0x40 [1488069.958243] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488069.966942] NMI watchdog: BUG: soft lockup - CPU#91 stuck for 23s! [ptlrpcd_05_04:54220] [1488069.966970] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488069.966977] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488069.966980] CPU: 91 PID: 54220 Comm: ptlrpcd_05_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488069.966981] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488069.966982] task: ffff8abaee63c100 ti: ffff8abaff6ec000 task.ti: ffff8abaff6ec000 [1488069.966985] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488069.966986] RSP: 0018:ffff8abaff6efb78 EFLAGS: 00000246 [1488069.966987] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002d90000 [1488069.966988] RDX: ffff8abaffb5b780 RSI: 0000000001710001 RDI: ffff8adafa7b1b00 [1488069.966989] RBP: ffff8abaff6efb78 R08: ffff8adaffcdb780 R09: 0000000000000000 [1488069.966990] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad86dcdb000 [1488069.966991] R13: 0005c4aa17a95ea0 R14: ffff8accdd36aa00 R15: 0000000000000007 [1488069.966993] FS: 00007ea1b9066700(0000) GS:ffff8adaffcc0000(0000) knlGS:0000000000000000 [1488069.966994] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488069.966995] CR2: 00007f8c1f871000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488069.966996] Call Trace: [1488069.966998] [] queued_spin_lock_slowpath+0xb/0xf [1488069.967000] [] _raw_spin_lock+0x20/0x30 [1488069.967008] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488069.967019] [] LNetMDUnlink+0x78/0x180 [lnet] [1488069.967058] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488069.967060] [] ? del_timer_sync+0x52/0x60 [1488069.967097] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488069.967134] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488069.967173] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488069.967175] [] ? wake_up_state+0x20/0x20 [1488069.967214] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488069.967215] [] kthread+0xd1/0xe0 [1488069.967217] [] ? insert_kthread_work+0x40/0x40 [1488069.967219] [] ret_from_fork_nospec_begin+0xe/0x21 [1488069.967221] [] ? insert_kthread_work+0x40/0x40 [1488069.967242] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488073.148153] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ptlrpcd_04_10:54214] [1488073.148182] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488073.148190] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488073.148192] CPU: 1 PID: 54214 Comm: ptlrpcd_04_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488073.148193] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488073.148194] task: ffff8abaf82cd140 ti: ffff8abaf4ad4000 task.ti: ffff8abaf4ad4000 [1488073.148198] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488073.148199] RSP: 0018:ffff8abaf4ad7b78 EFLAGS: 00000246 [1488073.148200] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000090000 [1488073.148201] RDX: ffff8acaffb9b780 RSI: 0000000001890001 RDI: ffff8adafa7b1b00 [1488073.148202] RBP: ffff8abaf4ad7b78 R08: ffff8acaffa1b780 R09: 0000000000000000 [1488073.148203] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac248730000 [1488073.148204] R13: 0005c4aa17c86390 R14: ffff8ac9ade81200 R15: 0000000000000007 [1488073.148206] FS: 00007f8eefb88700(0000) GS:ffff8acaffa00000(0000) knlGS:0000000000000000 [1488073.148207] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488073.148208] CR2: 00007f8eefd2e000 CR3: 0000004039b7e000 CR4: 00000000003407e0 [1488073.148209] Call Trace: [1488073.148211] [] queued_spin_lock_slowpath+0xb/0xf [1488073.148213] [] _raw_spin_lock+0x20/0x30 [1488073.148221] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488073.148231] [] LNetMDUnlink+0x78/0x180 [lnet] [1488073.148270] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488073.148272] [] ? del_timer_sync+0x52/0x60 [1488073.148310] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488073.148347] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488073.148386] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488073.148388] [] ? wake_up_state+0x20/0x20 [1488073.148427] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488073.148428] [] kthread+0xd1/0xe0 [1488073.148430] [] ? insert_kthread_work+0x40/0x40 [1488073.148432] [] ret_from_fork_nospec_begin+0xe/0x21 [1488073.148435] [] ? insert_kthread_work+0x40/0x40 [1488073.148455] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488073.393171] NMI watchdog: BUG: soft lockup - CPU#28 stuck for 23s! [ptlrpcd_02_05:54185] [1488073.393197] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488073.393205] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488073.393207] CPU: 28 PID: 54185 Comm: ptlrpcd_02_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488073.393209] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488073.393210] task: ffff8abaee644100 ti: ffff8abafa6f4000 task.ti: ffff8abafa6f4000 [1488073.393213] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488073.393214] RSP: 0018:ffff8abafa6f7b78 EFLAGS: 00000246 [1488073.393215] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000e10000 [1488073.393216] RDX: ffff8a8aff25b780 RSI: 0000000000410001 RDI: ffff8adafa7b1b00 [1488073.393217] RBP: ffff8abafa6f7b78 R08: ffff8aaaffadb780 R09: 0000000000000000 [1488073.393218] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa144f21200 [1488073.393219] R13: 0005c4aa180e0f10 R14: ffff8aa4fd689800 R15: 0000000000000007 [1488073.393221] FS: 00007ea1ba068700(0000) GS:ffff8aaaffac0000(0000) knlGS:0000000000000000 [1488073.393221] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488073.393222] CR2: 00007f8c22c61000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488073.393223] Call Trace: [1488073.393226] [] queued_spin_lock_slowpath+0xb/0xf [1488073.393228] [] _raw_spin_lock+0x20/0x30 [1488073.393235] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488073.393246] [] LNetMDUnlink+0x78/0x180 [lnet] [1488073.393284] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488073.393321] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488073.393358] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488073.393397] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488073.393399] [] ? wake_up_state+0x20/0x20 [1488073.393437] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488073.393439] [] kthread+0xd1/0xe0 [1488073.393441] [] ? insert_kthread_work+0x40/0x40 [1488073.393442] [] ret_from_fork_nospec_begin+0xe/0x21 [1488073.393444] [] ? insert_kthread_work+0x40/0x40 [1488073.393464] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488073.511177] NMI watchdog: BUG: soft lockup - CPU#41 stuck for 22s! [ptlrpcd_04_11:54215] [1488073.511204] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488073.511212] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488073.511214] CPU: 41 PID: 54215 Comm: ptlrpcd_04_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488073.511215] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488073.511216] task: ffff8abaf82ce180 ti: ffff8abafc750000 task.ti: ffff8abafc750000 [1488073.511220] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488073.511221] RSP: 0018:ffff8abafc753b78 EFLAGS: 00000246 [1488073.511222] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001490000 [1488073.511223] RDX: ffff8aaaffa5b780 RSI: 0000000000610001 RDI: ffff8adafa7b1b00 [1488073.511223] RBP: ffff8abafc753b78 R08: ffff8acaffb5b780 R09: 0000000000000000 [1488073.511224] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abbeeaa5a00 [1488073.511226] R13: 0005c4aa17cd2ea0 R14: ffff8abbeea05700 R15: 0000000000000007 [1488073.511227] FS: 00007ea1b0054700(0000) GS:ffff8acaffb40000(0000) knlGS:0000000000000000 [1488073.511228] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488073.511229] CR2: 00007f8c1b471000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488073.511230] Call Trace: [1488073.511233] [] queued_spin_lock_slowpath+0xb/0xf [1488073.511235] [] _raw_spin_lock+0x20/0x30 [1488073.511243] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488073.511253] [] LNetMDUnlink+0x78/0x180 [lnet] [1488073.511292] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488073.511329] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488073.511367] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488073.511406] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488073.511408] [] ? wake_up_state+0x20/0x20 [1488073.511446] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488073.511448] [] kthread+0xd1/0xe0 [1488073.511450] [] ? insert_kthread_work+0x40/0x40 [1488073.511452] [] ret_from_fork_nospec_begin+0xe/0x21 [1488073.511454] [] ? insert_kthread_work+0x40/0x40 [1488073.511474] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488073.621183] NMI watchdog: BUG: soft lockup - CPU#53 stuck for 22s! [ptlrpcd_06_09:54237] [1488073.621210] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488073.621218] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488073.621220] CPU: 53 PID: 54237 Comm: ptlrpcd_06_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488073.621221] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488073.621222] task: ffff8abafbc78000 ti: ffff8abafbbec000 task.ti: ffff8abafbbec000 [1488073.621226] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488073.621227] RSP: 0018:ffff8abafbbefb78 EFLAGS: 00000246 [1488073.621228] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001a90000 [1488073.621228] RDX: ffff8acaffc9b780 RSI: 0000000002890001 RDI: ffff8adafa7b1b00 [1488073.621230] RBP: ffff8abafbbefb78 R08: ffff8aeaffb9b780 R09: 0000000000000000 [1488073.621231] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ae286868200 [1488073.621232] R13: 0005c4aa18160ad0 R14: ffff8ae92e9aaa00 R15: 0000000000000007 [1488073.621234] FS: 00007ea1d60a0700(0000) GS:ffff8aeaffb80000(0000) knlGS:0000000000000000 [1488073.621235] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488073.621236] CR2: 00007eb6dd85a000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488073.621237] Call Trace: [1488073.621239] [] queued_spin_lock_slowpath+0xb/0xf [1488073.621241] [] _raw_spin_lock+0x20/0x30 [1488073.621250] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488073.621260] [] LNetMDUnlink+0x78/0x180 [lnet] [1488073.621299] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488073.621336] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488073.621373] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488073.621412] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488073.621414] [] ? wake_up_state+0x20/0x20 [1488073.621453] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488073.621454] [] kthread+0xd1/0xe0 [1488073.621456] [] ? insert_kthread_work+0x40/0x40 [1488073.621458] [] ret_from_fork_nospec_begin+0xe/0x21 [1488073.621460] [] ? insert_kthread_work+0x40/0x40 [1488073.621480] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488073.839199] NMI watchdog: BUG: soft lockup - CPU#77 stuck for 22s! [ptlrpcd_06_04:54232] [1488073.839226] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488073.839234] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488073.839236] CPU: 77 PID: 54232 Comm: ptlrpcd_06_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488073.839237] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488073.839239] task: ffff8abaf4782080 ti: ffff8abafbecc000 task.ti: ffff8abafbecc000 [1488073.839242] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488073.839243] RSP: 0018:ffff8abafbecfb78 EFLAGS: 00000246 [1488073.839244] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002690000 [1488073.839245] RDX: ffff8a8aff4db780 RSI: 0000000002c10001 RDI: ffff8adafa7b1b00 [1488073.839246] RBP: ffff8abafbecfb78 R08: ffff8aeaffc5b780 R09: 0000000000000000 [1488073.839247] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ae328ff7a00 [1488073.839248] R13: 0005c4aa17cbee60 R14: ffff8add42fe9800 R15: 0000000000000007 [1488073.839250] FS: 00007ea1d2098700(0000) GS:ffff8aeaffc40000(0000) knlGS:0000000000000000 [1488073.839250] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488073.839251] CR2: 00007fc0cc5b7198 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488073.839252] Call Trace: [1488073.839255] [] queued_spin_lock_slowpath+0xb/0xf [1488073.839257] [] _raw_spin_lock+0x20/0x30 [1488073.839265] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488073.839275] [] LNetMDUnlink+0x78/0x180 [lnet] [1488073.839314] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488073.839352] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488073.839389] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488073.839427] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488073.839430] [] ? wake_up_state+0x20/0x20 [1488073.839468] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488073.839470] [] kthread+0xd1/0xe0 [1488073.839472] [] ? insert_kthread_work+0x40/0x40 [1488073.839473] [] ret_from_fork_nospec_begin+0xe/0x21 [1488073.839476] [] ? insert_kthread_work+0x40/0x40 [1488073.839496] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488077.238424] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ptlrpcd_05_08:54224] [1488077.238452] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488077.238459] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488077.238462] CPU: 11 PID: 54224 Comm: ptlrpcd_05_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488077.238463] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488077.238464] task: ffff8abafb339040 ti: ffff8abaeca28000 task.ti: ffff8abaeca28000 [1488077.238467] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488077.238469] RSP: 0018:ffff8abaeca2bb78 EFLAGS: 00000246 [1488077.238469] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000590000 [1488077.238471] RDX: ffff8abaffc1b780 RSI: 0000000002310001 RDI: ffff8adafa7b1b00 [1488077.238471] RBP: ffff8abaeca2bb78 R08: ffff8adaffa5b780 R09: 0000000000000000 [1488077.238472] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad89cf94000 [1488077.238474] R13: 0005c4aa17a3b130 R14: ffff8ad165ebe300 R15: 0000000000000007 [1488077.238475] FS: 00007ea1b7062700(0000) GS:ffff8adaffa40000(0000) knlGS:0000000000000000 [1488077.238476] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488077.238477] CR2: 00007fbad3d83000 CR3: 0000002034d8c000 CR4: 00000000003407e0 [1488077.238478] Call Trace: [1488077.238480] [] queued_spin_lock_slowpath+0xb/0xf [1488077.238482] [] _raw_spin_lock+0x20/0x30 [1488077.238490] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488077.238501] [] LNetMDUnlink+0x78/0x180 [lnet] [1488077.238539] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488077.238542] [] ? del_timer_sync+0x52/0x60 [1488077.238579] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488077.238616] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488077.238655] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488077.238657] [] ? wake_up_state+0x20/0x20 [1488077.238696] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488077.238697] [] kthread+0xd1/0xe0 [1488077.238700] [] ? insert_kthread_work+0x40/0x40 [1488077.238702] [] ret_from_fork_nospec_begin+0xe/0x21 [1488077.238703] [] ? insert_kthread_work+0x40/0x40 [1488077.238723] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.165940] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [ptlrpcd_05_01:54217] [1488085.165964] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.165971] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.165973] CPU: 3 PID: 54217 Comm: ptlrpcd_05_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.165973] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.165974] task: ffff8abaee639040 ti: ffff8abafaf6c000 task.ti: ffff8abafaf6c000 [1488085.165978] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.165978] RSP: 0018:ffff8abafaf6fb78 EFLAGS: 00000246 [1488085.165979] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000190000 [1488085.165980] RDX: ffff8a8aff49b780 RSI: 0000000002810001 RDI: ffff8adafa7b1b00 [1488085.165980] RBP: ffff8abafaf6fb78 R08: ffff8adaffa1b780 R09: 0000000000000000 [1488085.165981] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad1c9f1aa00 [1488085.165982] R13: 0005c4aa17afa680 R14: ffff8adab82b8300 R15: 0000000000000007 [1488085.165983] FS: 00007ea1d2098700(0000) GS:ffff8adaffa00000(0000) knlGS:0000000000000000 [1488085.165984] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.165984] CR2: 00007fff8047d958 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.165985] Call Trace: [1488085.165987] [] queued_spin_lock_slowpath+0xb/0xf [1488085.165989] [] _raw_spin_lock+0x20/0x30 [1488085.165997] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.166007] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.166045] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.166083] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.166119] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.166158] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.166160] [] ? wake_up_state+0x20/0x20 [1488085.166199] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.166200] [] kthread+0xd1/0xe0 [1488085.166202] [] ? insert_kthread_work+0x40/0x40 [1488085.166204] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.166206] [] ? insert_kthread_work+0x40/0x40 [1488085.166225] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.383960] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 22s! [ptlrpcd_05_06:54222] [1488085.383988] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.383995] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.383998] CPU: 27 PID: 54222 Comm: ptlrpcd_05_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.383999] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.384000] task: ffff8abaee63e180 ti: ffff8abafdbdc000 task.ti: ffff8abafdbdc000 [1488085.384003] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.384004] RSP: 0018:ffff8abafdbdfb78 EFLAGS: 00000246 [1488085.384005] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000d90000 [1488085.384006] RDX: ffff8acaffa9b780 RSI: 0000000000890001 RDI: ffff8adafa7b1b00 [1488085.384007] RBP: ffff8abafdbdfb78 R08: ffff8adaffadb780 R09: 0000000000000000 [1488085.384008] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad8b63d6600 [1488085.384009] R13: 0005c4aa17d33040 R14: ffff8ad23dd66c00 R15: 0000000000000007 [1488085.384011] FS: 00007f1993824740(0000) GS:ffff8adaffac0000(0000) knlGS:0000000000000000 [1488085.384012] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.384013] CR2: 00007f8c1d455000 CR3: 0000005f8433a000 CR4: 00000000003407e0 [1488085.384013] Call Trace: [1488085.384016] [] queued_spin_lock_slowpath+0xb/0xf [1488085.384017] [] _raw_spin_lock+0x20/0x30 [1488085.384025] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.384036] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.384074] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.384112] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.384150] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.384188] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.384191] [] ? wake_up_state+0x20/0x20 [1488085.384229] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.384231] [] kthread+0xd1/0xe0 [1488085.384233] [] ? insert_kthread_work+0x40/0x40 [1488085.384234] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.384236] [] ? insert_kthread_work+0x40/0x40 [1488085.384256] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.402959] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [ptlrpcd_06_10:54238] [1488085.402987] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.402995] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.402997] CPU: 29 PID: 54238 Comm: ptlrpcd_06_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.402998] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.402999] task: ffff8abafbc79040 ti: ffff8abaf8e64000 task.ti: ffff8abaf8e64000 [1488085.403003] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488085.403004] RSP: 0018:ffff8abaf8e67b78 EFLAGS: 00000246 [1488085.403005] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000e90000 [1488085.403006] RDX: ffff8aeaffa5b780 RSI: 0000000000690001 RDI: ffff8adafa7b1b00 [1488085.403007] RBP: ffff8abaf8e67b78 R08: ffff8aeaffadb780 R09: 0000000000000000 [1488085.403008] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adbe028a600 [1488085.403009] R13: 0005c4aa17c61800 R14: ffff8ae33a3d4e00 R15: 0000000000000007 [1488085.403010] FS: 00007ea1b485d700(0000) GS:ffff8aeaffac0000(0000) knlGS:0000000000000000 [1488085.403011] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.403012] CR2: 00007f470eefd038 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.403013] Call Trace: [1488085.403016] [] queued_spin_lock_slowpath+0xb/0xf [1488085.403018] [] _raw_spin_lock+0x20/0x30 [1488085.403026] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.403037] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.403075] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.403113] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.403150] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.403188] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.403191] [] ? wake_up_state+0x20/0x20 [1488085.403229] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.403231] [] kthread+0xd1/0xe0 [1488085.403233] [] ? insert_kthread_work+0x40/0x40 [1488085.403235] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.403237] [] ? insert_kthread_work+0x40/0x40 [1488085.403257] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488085.548966] NMI watchdog: BUG: soft lockup - CPU#45 stuck for 22s! [ptlrpcd_06_00:54228] [1488085.548992] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.548999] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.549002] CPU: 45 PID: 54228 Comm: ptlrpcd_06_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.549003] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.549004] task: ffff8abafb33d140 ti: ffff8abafc584000 task.ti: ffff8abafc584000 [1488085.549007] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.549008] RSP: 0018:ffff8abafc587b78 EFLAGS: 00000246 [1488085.549009] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001690000 [1488085.549010] RDX: ffff8afb3f81b780 RSI: 0000000000390001 RDI: ffff8adafa7b1b00 [1488085.549011] RBP: ffff8abafc587b78 R08: ffff8aeaffb5b780 R09: 0000000000000000 [1488085.549012] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adc55a5da00 [1488085.549013] R13: 0005c4aa17c40f20 R14: ffff8ae89e598f00 R15: 0000000000000007 [1488085.549015] FS: 00007ea1b485d700(0000) GS:ffff8aeaffb40000(0000) knlGS:0000000000000000 [1488085.549016] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.549017] CR2: 00007eea0371e002 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.549018] Call Trace: [1488085.549020] [] queued_spin_lock_slowpath+0xb/0xf [1488085.549022] [] _raw_spin_lock+0x20/0x30 [1488085.549030] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.549041] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.549079] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.549117] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.549154] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.549193] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.549195] [] ? wake_up_state+0x20/0x20 [1488085.549234] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.549235] [] kthread+0xd1/0xe0 [1488085.549237] [] ? insert_kthread_work+0x40/0x40 [1488085.549239] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.549241] [] ? insert_kthread_work+0x40/0x40 [1488085.549261] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.593973] NMI watchdog: BUG: soft lockup - CPU#50 stuck for 22s! [ptlrpcd_01_02:54170] [1488085.594001] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.594009] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.594012] CPU: 50 PID: 54170 Comm: ptlrpcd_01_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.594013] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.594014] task: ffff8abaf63430c0 ti: ffff8abafa204000 task.ti: ffff8abafa204000 [1488085.594017] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488085.594018] RSP: 0018:ffff8abafa207b78 EFLAGS: 00000246 [1488085.594019] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001910000 [1488085.594020] RDX: ffff8aeaffb5b780 RSI: 0000000001690001 RDI: ffff8adafa7b1b00 [1488085.594021] RBP: ffff8abafa207b78 R08: ffff8a9affb9b780 R09: 0000000000000000 [1488085.594022] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a96d0ea1200 [1488085.594023] R13: 0005c4aa179a0f70 R14: ffff8a8de4f62d00 R15: 0000000000000007 [1488085.594025] FS: 00007ea1da0a8700(0000) GS:ffff8a9affb80000(0000) knlGS:0000000000000000 [1488085.594026] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.594027] CR2: 00007efcd3da6000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.594028] Call Trace: [1488085.594030] [] queued_spin_lock_slowpath+0xb/0xf [1488085.594032] [] _raw_spin_lock+0x20/0x30 [1488085.594040] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.594050] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.594088] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.594126] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.594163] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.594201] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.594203] [] ? wake_up_state+0x20/0x20 [1488085.594241] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.594243] [] kthread+0xd1/0xe0 [1488085.594245] [] ? insert_kthread_work+0x40/0x40 [1488085.594247] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.594249] [] ? insert_kthread_work+0x40/0x40 [1488085.594269] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488085.665973] NMI watchdog: BUG: soft lockup - CPU#58 stuck for 22s! [ptlrpcd_01_07:54175] [1488085.666001] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.666008] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.666011] CPU: 58 PID: 54175 Comm: ptlrpcd_01_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.666012] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.666014] task: ffff8abaf4da9040 ti: ffff8abaee6dc000 task.ti: ffff8abaee6dc000 [1488085.666017] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.666018] RSP: 0018:ffff8abaee6dfb78 EFLAGS: 00000246 [1488085.666019] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001d10000 [1488085.666020] RDX: ffff8afb3f89b780 RSI: 0000000000b90001 RDI: ffff8adafa7b1b00 [1488085.666021] RBP: ffff8abaee6dfb78 R08: ffff8a9affbdb780 R09: 0000000000000000 [1488085.666022] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a950c268600 [1488085.666023] R13: 0005c4aa17a59da0 R14: ffff8a8c35a04e00 R15: 0000000000000007 [1488085.666025] FS: 00007ea1ba869700(0000) GS:ffff8a9affbc0000(0000) knlGS:0000000000000000 [1488085.666026] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.666027] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.666028] Call Trace: [1488085.666030] [] queued_spin_lock_slowpath+0xb/0xf [1488085.666032] [] _raw_spin_lock+0x20/0x30 [1488085.666040] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.666051] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.666089] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.666127] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.666164] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.666203] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.666205] [] ? wake_up_state+0x20/0x20 [1488085.666243] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.666245] [] kthread+0xd1/0xe0 [1488085.666247] [] ? insert_kthread_work+0x40/0x40 [1488085.666249] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.666251] [] ? insert_kthread_work+0x40/0x40 [1488085.666271] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.675975] NMI watchdog: BUG: soft lockup - CPU#59 stuck for 22s! [ptlrpcd_05_00:54216] [1488085.676004] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.676011] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.676014] CPU: 59 PID: 54216 Comm: ptlrpcd_05_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.676015] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.676016] task: ffff8abaee638000 ti: ffff8abaf8220000 task.ti: ffff8abaf8220000 [1488085.676019] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488085.676020] RSP: 0018:ffff8abaf8223b78 EFLAGS: 00000246 [1488085.676021] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001d90000 [1488085.676022] RDX: ffff8a9affb5b780 RSI: 0000000001510001 RDI: ffff8adafa7b1b00 [1488085.676023] RBP: ffff8abaf8223b78 R08: ffff8adaffbdb780 R09: 0000000000000000 [1488085.676024] R10: 0000000000000000 R11: 000000000000000f R12: ffff8acd69a63c00 [1488085.676025] R13: 0005c4aa178827a0 R14: ffff8ad241ea4200 R15: 0000000000000007 [1488085.676027] FS: 00007f8c263ff700(0000) GS:ffff8adaffbc0000(0000) knlGS:0000000000000000 [1488085.676027] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.676029] CR2: 000000000230d6a8 CR3: 00000057926fe000 CR4: 00000000003407e0 [1488085.676029] Call Trace: [1488085.676031] [] queued_spin_lock_slowpath+0xb/0xf [1488085.676034] [] _raw_spin_lock+0x20/0x30 [1488085.676041] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.676052] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.676090] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.676128] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.676165] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.676204] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.676206] [] ? wake_up_state+0x20/0x20 [1488085.676244] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.676246] [] kthread+0xd1/0xe0 [1488085.676248] [] ? insert_kthread_work+0x40/0x40 [1488085.676250] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.676252] [] ? insert_kthread_work+0x40/0x40 [1488085.676272] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488085.712977] NMI watchdog: BUG: soft lockup - CPU#63 stuck for 22s! [ptlrpcd_07_06:54246] [1488085.713005] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.713012] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.713016] CPU: 63 PID: 54246 Comm: ptlrpcd_07_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.713018] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.713019] task: ffff8abafc522080 ti: ffff8abaf4f54000 task.ti: ffff8abaf4f54000 [1488085.713022] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.713023] RSP: 0018:ffff8abaf4f57b78 EFLAGS: 00000246 [1488085.713024] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001f90000 [1488085.713025] RDX: ffff8a9affa9b780 RSI: 0000000000910000 RDI: ffff8adafa7b1b00 [1488085.713026] RBP: ffff8abaf4f57b78 R08: ffff8afb3f9db780 R09: 0000000000000000 [1488085.713027] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aed569e2a00 [1488085.713028] R13: 0005c4aa17c51860 R14: ffff8aed24f64b00 R15: 0000000000000007 [1488085.713030] FS: 00007ea1d389b700(0000) GS:ffff8afb3f9c0000(0000) knlGS:0000000000000000 [1488085.713031] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.713032] CR2: 00007ecf3e8c7ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.713033] Call Trace: [1488085.713035] [] queued_spin_lock_slowpath+0xb/0xf [1488085.713037] [] _raw_spin_lock+0x20/0x30 [1488085.713045] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.713055] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.713094] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.713132] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.713169] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.713208] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.713209] [] ? wake_up_state+0x20/0x20 [1488085.713248] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.713250] [] kthread+0xd1/0xe0 [1488085.713252] [] ? insert_kthread_work+0x40/0x40 [1488085.713254] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.713256] [] ? insert_kthread_work+0x40/0x40 [1488085.713276] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.759984] NMI watchdog: BUG: soft lockup - CPU#68 stuck for 22s! [ptlrpcd_02_02:54182] [1488085.760013] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.760020] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.760023] CPU: 68 PID: 54182 Comm: ptlrpcd_02_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.760024] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.760025] task: ffff8abaee641040 ti: ffff8abafcdf0000 task.ti: ffff8abafcdf0000 [1488085.760028] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.760029] RSP: 0018:ffff8abafcdf3b78 EFLAGS: 00000246 [1488085.760030] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002210000 [1488085.760031] RDX: ffff8aaaffbdb780 RSI: 0000000001e10001 RDI: ffff8adafa7b1b00 [1488085.760033] RBP: ffff8abafcdf3b78 R08: ffff8aaaffc1b780 R09: 0000000000000000 [1488085.760033] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa4ef142800 [1488085.760035] R13: 0005c4aa17cce1a0 R14: ffff8aa4721f3900 R15: 0000000000000007 [1488085.760036] FS: 00007ea149112700(0000) GS:ffff8aaaffc00000(0000) knlGS:0000000000000000 [1488085.760037] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.760038] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.760039] Call Trace: [1488085.760041] [] queued_spin_lock_slowpath+0xb/0xf [1488085.760043] [] _raw_spin_lock+0x20/0x30 [1488085.760051] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.760062] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.760100] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.760138] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.760175] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.760213] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.760216] [] ? wake_up_state+0x20/0x20 [1488085.760254] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.760256] [] kthread+0xd1/0xe0 [1488085.760258] [] ? insert_kthread_work+0x40/0x40 [1488085.760260] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.760262] [] ? insert_kthread_work+0x40/0x40 [1488085.760282] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.902994] NMI watchdog: BUG: soft lockup - CPU#84 stuck for 22s! [ptlrpcd_02_00:54180] [1488085.903018] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.903026] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.903028] CPU: 84 PID: 54180 Comm: ptlrpcd_02_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.903029] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.903031] task: ffff8abaf4dae180 ti: ffff8abaecadc000 task.ti: ffff8abaecadc000 [1488085.903034] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488085.903035] RSP: 0018:ffff8abaecadfb78 EFLAGS: 00000246 [1488085.903036] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002a10000 [1488085.903037] RDX: ffff8aaaffb1b780 RSI: 0000000001210001 RDI: ffff8adafa7b1b00 [1488085.903037] RBP: ffff8abaecadfb78 R08: ffff8aaaffc9b780 R09: 0000000000000000 [1488085.903039] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9bffcd7200 [1488085.903040] R13: 0005c4aa17cd8670 R14: ffff8aa639e9aa00 R15: 0000000000000007 [1488085.903041] FS: 00007ea1c8885700(0000) GS:ffff8aaaffc80000(0000) knlGS:0000000000000000 [1488085.903042] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.903043] CR2: 00007ee6f34aa006 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.903044] Call Trace: [1488085.903046] [] queued_spin_lock_slowpath+0xb/0xf [1488085.903048] [] _raw_spin_lock+0x20/0x30 [1488085.903056] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.903067] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.903104] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.903142] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.903179] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.903217] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.903219] [] ? wake_up_state+0x20/0x20 [1488085.903257] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.903259] [] kthread+0xd1/0xe0 [1488085.903261] [] ? insert_kthread_work+0x40/0x40 [1488085.903263] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.903264] [] ? insert_kthread_work+0x40/0x40 [1488085.903284] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488085.994995] NMI watchdog: BUG: soft lockup - CPU#94 stuck for 23s! [ptlrpcd_03_02:54194] [1488085.995022] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488085.995030] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488085.995032] CPU: 94 PID: 54194 Comm: ptlrpcd_03_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488085.995034] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488085.995035] task: ffff8abaebafe180 ti: ffff8abaf8e10000 task.ti: ffff8abaf8e10000 [1488085.995038] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488085.995039] RSP: 0018:ffff8abaf8e13b78 EFLAGS: 00000246 [1488085.995040] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002f10000 [1488085.995041] RDX: ffff8a8aff2db780 RSI: 0000000000c10001 RDI: ffff8adafa7b1b00 [1488085.995042] RBP: ffff8abaf8e13b78 R08: ffff8abaffcdb780 R09: 0000000000000000 [1488085.995043] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab1562e0600 [1488085.995044] R13: 0005c4aa17cca680 R14: ffff8ab8fa384b00 R15: 0000000000000007 [1488085.995046] FS: 00007ea1d1897700(0000) GS:ffff8abaffcc0000(0000) knlGS:0000000000000000 [1488085.995047] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488085.995048] CR2: 00007f47fafb9014 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488085.995049] Call Trace: [1488085.995051] [] queued_spin_lock_slowpath+0xb/0xf [1488085.995054] [] _raw_spin_lock+0x20/0x30 [1488085.995061] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488085.995072] [] LNetMDUnlink+0x78/0x180 [lnet] [1488085.995110] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488085.995147] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488085.995184] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488085.995222] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488085.995224] [] ? wake_up_state+0x20/0x20 [1488085.995263] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488085.995264] [] kthread+0xd1/0xe0 [1488085.995266] [] ? insert_kthread_work+0x40/0x40 [1488085.995268] [] ret_from_fork_nospec_begin+0xe/0x21 [1488085.995270] [] ? insert_kthread_work+0x40/0x40 [1488085.995290] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.157208] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ptlrpcd_01_10:54178] [1488089.157234] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.157242] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.157244] CPU: 2 PID: 54178 Comm: ptlrpcd_01_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.157245] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.157247] task: ffff8abaf4dac100 ti: ffff8abafd6fc000 task.ti: ffff8abafd6fc000 [1488089.157250] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488089.157251] RSP: 0018:ffff8abafd6ffb78 EFLAGS: 00000246 [1488089.157252] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000110000 [1488089.157253] RDX: ffff8a9affc1b780 RSI: 0000000002110001 RDI: ffff8adafa7b1b00 [1488089.157254] RBP: ffff8abafd6ffb78 R08: ffff8a9affa1b780 R09: 0000000000000000 [1488089.157255] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a94eddae200 [1488089.157256] R13: 0005c4aa179f63f0 R14: ffff8a97eef74200 R15: 0000000000000007 [1488089.157258] FS: 00007ea1cb08a700(0000) GS:ffff8a9affa00000(0000) knlGS:0000000000000000 [1488089.157259] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.157260] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.157261] Call Trace: [1488089.157263] [] queued_spin_lock_slowpath+0xb/0xf [1488089.157265] [] _raw_spin_lock+0x20/0x30 [1488089.157273] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.157284] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.157321] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.157359] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.157395] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.157434] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.157436] [] ? wake_up_state+0x20/0x20 [1488089.157475] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.157477] [] kthread+0xd1/0xe0 [1488089.157479] [] ? insert_kthread_work+0x40/0x40 [1488089.157480] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.157482] [] ? insert_kthread_work+0x40/0x40 [1488089.157503] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488089.176208] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [ptlrpcd_02_10:54190] [1488089.176235] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.176242] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.176245] CPU: 4 PID: 54190 Comm: ptlrpcd_02_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.176245] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.176247] task: ffff8abaebafa080 ti: ffff8abaeca8c000 task.ti: ffff8abaeca8c000 [1488089.176250] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.176251] RSP: 0018:ffff8abaeca8fb78 EFLAGS: 00000246 [1488089.176252] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000210000 [1488089.176253] RDX: ffff8adaffc9b780 RSI: 0000000002990001 RDI: ffff8adafa7b1b00 [1488089.176254] RBP: ffff8abaeca8fb78 R08: ffff8aaaffa1b780 R09: 0000000000000000 [1488089.176255] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8fde494e00 [1488089.176256] R13: 0005c4aa17cd4900 R14: ffff8aa3ff308000 R15: 0000000000000007 [1488089.176258] FS: 00007ea1d509e700(0000) GS:ffff8aaaffa00000(0000) knlGS:0000000000000000 [1488089.176258] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.176260] CR2: 0000000000453b20 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.176260] Call Trace: [1488089.176263] [] queued_spin_lock_slowpath+0xb/0xf [1488089.176265] [] _raw_spin_lock+0x20/0x30 [1488089.176273] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.176284] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.176322] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.176361] [] ? deuuidify+0x90/0x90 [ptlrpc] [1488089.176399] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.176437] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.176476] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.176478] [] ? wake_up_state+0x20/0x20 [1488089.176516] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.176518] [] kthread+0xd1/0xe0 [1488089.176520] [] ? insert_kthread_work+0x40/0x40 [1488089.176522] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.176524] [] ? insert_kthread_work+0x40/0x40 [1488089.176544] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.184210] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [ptlrpcd_06_01:54229] [1488089.184236] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.184244] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.184246] CPU: 5 PID: 54229 Comm: ptlrpcd_06_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.184247] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.184248] task: ffff8abafb33e180 ti: ffff8abaf5350000 task.ti: ffff8abaf5350000 [1488089.184252] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488089.184253] RSP: 0018:ffff8abaf5353b78 EFLAGS: 00000246 [1488089.184254] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000290000 [1488089.184255] RDX: ffff8aeaffcdb780 RSI: 0000000002e90001 RDI: ffff8adafa7b1b00 [1488089.184256] RBP: ffff8abaf5353b78 R08: ffff8aeaffa1b780 R09: 0000000000000000 [1488089.184257] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aea9ae2e200 [1488089.184258] R13: 0005c4aa17d2e8f0 R14: ffff8ae450b52100 R15: 0000000000000007 [1488089.184260] FS: 00007ea1bb06a700(0000) GS:ffff8aeaffa00000(0000) knlGS:0000000000000000 [1488089.184261] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.184262] CR2: 00007f6e7e5bf590 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.184263] Call Trace: [1488089.184265] [] queued_spin_lock_slowpath+0xb/0xf [1488089.184267] [] _raw_spin_lock+0x20/0x30 [1488089.184275] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.184286] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.184324] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.184362] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.184399] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.184438] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.184440] [] ? wake_up_state+0x20/0x20 [1488089.184479] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.184480] [] kthread+0xd1/0xe0 [1488089.184482] [] ? insert_kthread_work+0x40/0x40 [1488089.184484] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.184487] [] ? insert_kthread_work+0x40/0x40 [1488089.184506] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488089.193211] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ptlrpcd_03_11:54203] [1488089.193240] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.193247] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.193250] CPU: 6 PID: 54203 Comm: ptlrpcd_03_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.193251] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.193252] task: ffff8aaba9651040 ti: ffff8abaed288000 task.ti: ffff8abaed288000 [1488089.193256] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.193256] RSP: 0018:ffff8abaed28bb78 EFLAGS: 00000246 [1488089.193257] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000310000 [1488089.193259] RDX: ffff8a9affadb780 RSI: 0000000000d10001 RDI: ffff8adafa7b1b00 [1488089.193260] RBP: ffff8abaed28bb78 R08: ffff8abaffa1b780 R09: 0000000000000000 [1488089.193260] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab36218f200 [1488089.193262] R13: 0005c4aa17d36130 R14: ffff8ab5a74fbc00 R15: 0000000000000007 [1488089.193264] FS: 00007ea1dc8ad700(0000) GS:ffff8abaffa00000(0000) knlGS:0000000000000000 [1488089.193265] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.193266] CR2: 00007f5ff48d9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.193266] Call Trace: [1488089.193269] [] queued_spin_lock_slowpath+0xb/0xf [1488089.193271] [] _raw_spin_lock+0x20/0x30 [1488089.193279] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.193290] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.193327] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.193365] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.193402] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.193440] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.193443] [] ? wake_up_state+0x20/0x20 [1488089.193480] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.193483] [] kthread+0xd1/0xe0 [1488089.193484] [] ? insert_kthread_work+0x40/0x40 [1488089.193486] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.193488] [] ? insert_kthread_work+0x40/0x40 [1488089.193509] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.203212] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ptlrpcd_07_03:54243] [1488089.203241] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.203248] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.203251] CPU: 7 PID: 54243 Comm: ptlrpcd_07_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.203252] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.203253] task: ffff8abafbc7e180 ti: ffff8abafbb24000 task.ti: ffff8abafbb24000 [1488089.203257] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.203258] RSP: 0018:ffff8abafbb27b78 EFLAGS: 00000246 [1488089.203259] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000390000 [1488089.203260] RDX: ffff8acaffcdb780 RSI: 0000000002c90001 RDI: ffff8adafa7b1b00 [1488089.203261] RBP: ffff8abafbb27b78 R08: ffff8afb3f81b780 R09: 0000000000000000 [1488089.203262] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af1a1f2a000 [1488089.203263] R13: 0005c4aa17ccfa80 R14: ffff8af090a4da00 R15: 0000000000000007 [1488089.203265] FS: 00007ea1ca889700(0000) GS:ffff8afb3f800000(0000) knlGS:0000000000000000 [1488089.203265] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.203267] CR2: 00007ee55cb4bff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.203267] Call Trace: [1488089.203270] [] queued_spin_lock_slowpath+0xb/0xf [1488089.203272] [] _raw_spin_lock+0x20/0x30 [1488089.203280] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.203290] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.203329] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.203366] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.203404] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.203443] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.203445] [] ? wake_up_state+0x20/0x20 [1488089.203483] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.203485] [] kthread+0xd1/0xe0 [1488089.203487] [] ? insert_kthread_work+0x40/0x40 [1488089.203489] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.203491] [] ? insert_kthread_work+0x40/0x40 [1488089.203511] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.211207] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [ptlrpcd_00_02:54158] [1488089.211235] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.211243] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.211245] CPU: 8 PID: 54158 Comm: ptlrpcd_00_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.211246] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.211247] task: ffff8afaf5f330c0 ti: ffff8abaf6878000 task.ti: ffff8abaf6878000 [1488089.211251] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.211252] RSP: 0018:ffff8abaf687bb78 EFLAGS: 00000246 [1488089.211253] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000410000 [1488089.211253] RDX: ffff8aeaffb1b780 RSI: 0000000001290001 RDI: ffff8adafa7b1b00 [1488089.211254] RBP: ffff8abaf687bb78 R08: ffff8a8aff25b780 R09: 0000000000000000 [1488089.211256] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8bc6590200 [1488089.211256] R13: 0005c4aa187a08a0 R14: ffff8a9a09e6da00 R15: 0000000000000007 [1488089.211258] FS: 00007ea1c9086700(0000) GS:ffff8a8aff240000(0000) knlGS:0000000000000000 [1488089.211259] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.211261] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.211261] Call Trace: [1488089.211263] [] queued_spin_lock_slowpath+0xb/0xf [1488089.211266] [] _raw_spin_lock+0x20/0x30 [1488089.211274] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.211284] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.211322] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.211360] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.211396] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.211435] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.211437] [] ? wake_up_state+0x20/0x20 [1488089.211475] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.211477] [] kthread+0xd1/0xe0 [1488089.211478] [] ? insert_kthread_work+0x40/0x40 [1488089.211481] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.211483] [] ? insert_kthread_work+0x40/0x40 [1488089.211503] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.221210] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ptlrpcd_04_07:54211] [1488089.221236] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.221244] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.221247] CPU: 9 PID: 54211 Comm: ptlrpcd_04_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.221247] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.221249] task: ffff8abaf82ca080 ti: ffff8abafd344000 task.ti: ffff8abafd344000 [1488089.221252] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.221253] RSP: 0018:ffff8abafd347b78 EFLAGS: 00000246 [1488089.221254] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000490000 [1488089.221255] RDX: ffff8acaffc5b780 RSI: 0000000002490001 RDI: ffff8adafa7b1b00 [1488089.221256] RBP: ffff8abafd347b78 R08: ffff8acaffa5b780 R09: 0000000000000000 [1488089.221257] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abc1b613c00 [1488089.221258] R13: 0005c4aa17cc1560 R14: ffff8ac421352a00 R15: 0000000000000007 [1488089.221260] FS: 00007ea1b0054700(0000) GS:ffff8acaffa40000(0000) knlGS:0000000000000000 [1488089.221261] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.221262] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.221263] Call Trace: [1488089.221265] [] queued_spin_lock_slowpath+0xb/0xf [1488089.221266] [] _raw_spin_lock+0x20/0x30 [1488089.221275] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.221285] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.221324] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.221362] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.221399] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.221437] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.221440] [] ? wake_up_state+0x20/0x20 [1488089.221478] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.221480] [] kthread+0xd1/0xe0 [1488089.221482] [] ? insert_kthread_work+0x40/0x40 [1488089.221484] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.221486] [] ? insert_kthread_work+0x40/0x40 [1488089.221506] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.231208] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [ptlrpcd_01_05:54173] [1488089.231237] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.231245] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.231248] CPU: 10 PID: 54173 Comm: ptlrpcd_01_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.231248] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.231250] task: ffff8abaf6346180 ti: ffff8abaf3da8000 task.ti: ffff8abaf3da8000 [1488089.231253] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.231254] RSP: 0018:ffff8abaf3dabb78 EFLAGS: 00000246 [1488089.231255] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000510000 [1488089.231256] RDX: ffff8afb3fa1b780 RSI: 0000000002390001 RDI: ffff8adafa7b1b00 [1488089.231257] RBP: ffff8abaf3dabb78 R08: ffff8a9affa5b780 R09: 0000000000000000 [1488089.231258] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a90a36b8a00 [1488089.231259] R13: 0005c4aa17a6f150 R14: ffff8a90848c4200 R15: 0000000000000007 [1488089.231261] FS: 00007ea1b2859700(0000) GS:ffff8a9affa40000(0000) knlGS:0000000000000000 [1488089.231261] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.231263] CR2: 00007ee89ec1e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.231263] Call Trace: [1488089.231266] [] queued_spin_lock_slowpath+0xb/0xf [1488089.231268] [] _raw_spin_lock+0x20/0x30 [1488089.231275] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.231286] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.231324] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.231362] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.231399] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.231437] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.231439] [] ? wake_up_state+0x20/0x20 [1488089.231477] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.231479] [] kthread+0xd1/0xe0 [1488089.231481] [] ? insert_kthread_work+0x40/0x40 [1488089.231483] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.231485] [] ? insert_kthread_work+0x40/0x40 [1488089.231505] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.248210] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ptlrpcd_02_03:54183] [1488089.248236] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.248244] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.248246] CPU: 12 PID: 54183 Comm: ptlrpcd_02_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.248247] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.248248] task: ffff8abaee642080 ti: ffff8abaf4528000 task.ti: ffff8abaf4528000 [1488089.248252] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.248253] RSP: 0018:ffff8abaf452bb78 EFLAGS: 00000246 [1488089.248254] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000610000 [1488089.248254] RDX: ffff8adaffb5b780 RSI: 0000000001590001 RDI: ffff8adafa7b1b00 [1488089.248255] RBP: ffff8abaf452bb78 R08: ffff8aaaffa5b780 R09: 0000000000000000 [1488089.248257] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9f7c35a000 [1488089.248258] R13: 0005c4aa17cf54e0 R14: ffff8aa6838d2a00 R15: 0000000000000007 [1488089.248260] FS: 00007fc0d2114880(0000) GS:ffff8aaaffa40000(0000) knlGS:0000000000000000 [1488089.248260] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.248262] CR2: 00007fc0cc4ae000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.248262] Call Trace: [1488089.248264] [] queued_spin_lock_slowpath+0xb/0xf [1488089.248266] [] _raw_spin_lock+0x20/0x30 [1488089.248274] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.248285] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.248323] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.248360] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.248397] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.248435] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.248437] [] ? wake_up_state+0x20/0x20 [1488089.248476] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.248477] [] kthread+0xd1/0xe0 [1488089.248479] [] ? insert_kthread_work+0x40/0x40 [1488089.248481] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.248483] [] ? insert_kthread_work+0x40/0x40 [1488089.248503] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.258212] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ptlrpcd_06_02:54230] [1488089.258239] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.258247] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.258250] CPU: 13 PID: 54230 Comm: ptlrpcd_06_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.258251] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.258252] task: ffff8abaf4780000 ti: ffff8abaf3c14000 task.ti: ffff8abaf3c14000 [1488089.258255] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.258256] RSP: 0018:ffff8abaf3c17b78 EFLAGS: 00000246 [1488089.258257] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000690000 [1488089.258258] RDX: ffff8afb3fadb780 RSI: 0000000002f90001 RDI: ffff8adafa7b1b00 [1488089.258259] RBP: ffff8abaf3c17b78 R08: ffff8aeaffa5b780 R09: 0000000000000000 [1488089.258260] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aea78775000 [1488089.258261] R13: 0005c4aa17ca1d80 R14: ffff8ae05a09b300 R15: 0000000000000007 [1488089.258263] FS: 00007ea1d0094700(0000) GS:ffff8aeaffa40000(0000) knlGS:0000000000000000 [1488089.258264] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.258265] CR2: 00007eeb9290a018 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.258266] Call Trace: [1488089.258268] [] queued_spin_lock_slowpath+0xb/0xf [1488089.258270] [] _raw_spin_lock+0x20/0x30 [1488089.258278] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.258289] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.258327] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.258365] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.258402] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.258441] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.258444] [] ? wake_up_state+0x20/0x20 [1488089.258482] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.258484] [] kthread+0xd1/0xe0 [1488089.258486] [] ? insert_kthread_work+0x40/0x40 [1488089.258488] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.258490] [] ? insert_kthread_work+0x40/0x40 [1488089.258510] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.267212] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [ptlrpcd_03_00:54192] [1488089.267240] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.267248] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.267250] CPU: 14 PID: 54192 Comm: ptlrpcd_03_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.267252] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.267253] task: ffff8abaebafc100 ti: ffff8abaecb70000 task.ti: ffff8abaecb70000 [1488089.267256] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.267257] RSP: 0018:ffff8abaecb73b78 EFLAGS: 00000246 [1488089.267258] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000710000 [1488089.267259] RDX: ffff8a9affc9b780 RSI: 0000000002910001 RDI: ffff8adafa7b1b00 [1488089.267260] RBP: ffff8abaecb73b78 R08: ffff8abaffa5b780 R09: 0000000000000000 [1488089.267261] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab181efec00 [1488089.267262] R13: 0005c4aa17cb8650 R14: ffff8ab98a532700 R15: 0000000000000007 [1488089.267264] FS: 00007ea1d309a700(0000) GS:ffff8abaffa40000(0000) knlGS:0000000000000000 [1488089.267265] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.267266] CR2: 00007ef9d5eb2334 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.267267] Call Trace: [1488089.267269] [] queued_spin_lock_slowpath+0xb/0xf [1488089.267271] [] _raw_spin_lock+0x20/0x30 [1488089.267279] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.267290] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.267328] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.267366] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.267403] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.267442] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.267444] [] ? wake_up_state+0x20/0x20 [1488089.267483] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.267484] [] kthread+0xd1/0xe0 [1488089.267486] [] ? insert_kthread_work+0x40/0x40 [1488089.267488] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.267490] [] ? insert_kthread_work+0x40/0x40 [1488089.267510] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.275212] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ptlrpcd_07_10:54250] [1488089.275239] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.275247] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.275249] CPU: 15 PID: 54250 Comm: ptlrpcd_07_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.275250] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.275251] task: ffff8abafc526180 ti: ffff8abafbe28000 task.ti: ffff8abafbe28000 [1488089.275254] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.275255] RSP: 0018:ffff8abafbe2bb78 EFLAGS: 00000246 [1488089.275256] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000790000 [1488089.275257] RDX: ffff8aaaffc5b780 RSI: 0000000002610001 RDI: ffff8adafa7b1b00 [1488089.275258] RBP: ffff8abafbe2bb78 R08: ffff8afb3f85b780 R09: 0000000000000000 [1488089.275259] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af5d3316800 [1488089.275260] R13: 0005c4aa17c32eb0 R14: ffff8af0e4b25400 R15: 0000000000000007 [1488089.275262] FS: 00007ea1b9066700(0000) GS:ffff8afb3f840000(0000) knlGS:0000000000000000 [1488089.275263] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.275264] CR2: 00007ecf3e917ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.275265] Call Trace: [1488089.275267] [] queued_spin_lock_slowpath+0xb/0xf [1488089.275269] [] _raw_spin_lock+0x20/0x30 [1488089.275277] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.275287] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.275326] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.275364] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.275401] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.275440] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.275442] [] ? wake_up_state+0x20/0x20 [1488089.275481] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.275482] [] kthread+0xd1/0xe0 [1488089.275484] [] ? insert_kthread_work+0x40/0x40 [1488089.275486] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.275488] [] ? insert_kthread_work+0x40/0x40 [1488089.275508] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.284215] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [ptlrpcd_00_07:54163] [1488089.284243] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.284251] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.284253] CPU: 16 PID: 54163 Comm: ptlrpcd_00_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.284254] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.284255] task: ffff8abaf815b0c0 ti: ffff8abaf8e1c000 task.ti: ffff8abaf8e1c000 [1488089.284259] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.284259] RSP: 0018:ffff8abaf8e1fb78 EFLAGS: 00000246 [1488089.284260] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000810000 [1488089.284261] RDX: ffff8adaffadb780 RSI: 0000000000d90001 RDI: ffff8adafa7b1b00 [1488089.284263] RBP: ffff8abaf8e1fb78 R08: ffff8a8aff29b780 R09: 0000000000000000 [1488089.284264] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8bc6592200 [1488089.284264] R13: 0005c4aa187a0d10 R14: ffff8a9a09e6bf00 R15: 0000000000000007 [1488089.284266] FS: 00007ea1d309a700(0000) GS:ffff8a8aff280000(0000) knlGS:0000000000000000 [1488089.284267] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.284268] CR2: 00007f47239c9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.284269] Call Trace: [1488089.284271] [] queued_spin_lock_slowpath+0xb/0xf [1488089.284273] [] _raw_spin_lock+0x20/0x30 [1488089.284282] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.284292] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.284330] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.284368] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.284406] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.284444] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.284446] [] ? wake_up_state+0x20/0x20 [1488089.284485] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.284487] [] kthread+0xd1/0xe0 [1488089.284489] [] ? insert_kthread_work+0x40/0x40 [1488089.284491] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.284493] [] ? insert_kthread_work+0x40/0x40 [1488089.284513] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.293217] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ptlrpcd_04_05:54209] [1488089.293245] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.293253] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.293255] CPU: 17 PID: 54209 Comm: ptlrpcd_04_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.293256] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.293257] task: ffff8abaf82c8000 ti: ffff8abafcde8000 task.ti: ffff8abafcde8000 [1488089.293261] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.293262] RSP: 0018:ffff8abafcdebb78 EFLAGS: 00000246 [1488089.293263] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000890000 [1488089.293264] RDX: ffff8aeaffb9b780 RSI: 0000000001a90001 RDI: ffff8adafa7b1b00 [1488089.293265] RBP: ffff8abafcdebb78 R08: ffff8acaffa9b780 R09: 0000000000000000 [1488089.293266] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac1a1ad1800 [1488089.293267] R13: 0005c4aa17cc9500 R14: ffff8ac1a1b0b300 R15: 0000000000000007 [1488089.293269] FS: 00007ea1be871700(0000) GS:ffff8acaffa80000(0000) knlGS:0000000000000000 [1488089.293270] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.293271] CR2: 00007ef98c7b802a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.293272] Call Trace: [1488089.293274] [] queued_spin_lock_slowpath+0xb/0xf [1488089.293276] [] _raw_spin_lock+0x20/0x30 [1488089.293284] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.293295] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.293333] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.293371] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.293408] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.293447] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.293449] [] ? wake_up_state+0x20/0x20 [1488089.293488] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.293490] [] kthread+0xd1/0xe0 [1488089.293491] [] ? insert_kthread_work+0x40/0x40 [1488089.293493] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.293495] [] ? insert_kthread_work+0x40/0x40 [1488089.293516] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.303216] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [ptlrpcd_01_09:54177] [1488089.303243] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.303251] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.303254] CPU: 18 PID: 54177 Comm: ptlrpcd_01_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.303254] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.303256] task: ffff8abaf4dab0c0 ti: ffff8abaf9440000 task.ti: ffff8abaf9440000 [1488089.303259] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.303260] RSP: 0018:ffff8abaf9443b78 EFLAGS: 00000246 [1488089.303261] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000910000 [1488089.303262] RDX: ffff8acaffbdb780 RSI: 0000000001c90001 RDI: ffff8adafa7b1b00 [1488089.303263] RBP: ffff8abaf9443b78 R08: ffff8a9affa9b780 R09: 0000000000000000 [1488089.303264] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a931aadea00 [1488089.303265] R13: 0005c4aa17a58110 R14: ffff8a8c35a01800 R15: 0000000000000007 [1488089.303267] FS: 00007ea1d409c700(0000) GS:ffff8a9affa80000(0000) knlGS:0000000000000000 [1488089.303268] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.303269] CR2: 00007fbad3d81000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.303270] Call Trace: [1488089.303272] [] queued_spin_lock_slowpath+0xb/0xf [1488089.303274] [] _raw_spin_lock+0x20/0x30 [1488089.303282] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.303293] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.303331] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.303368] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.303405] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.303445] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.303447] [] ? wake_up_state+0x20/0x20 [1488089.303486] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.303488] [] kthread+0xd1/0xe0 [1488089.303490] [] ? insert_kthread_work+0x40/0x40 [1488089.303492] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.303493] [] ? insert_kthread_work+0x40/0x40 [1488089.303514] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.320217] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ptlrpcd_02_07:54187] [1488089.320243] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.320250] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.320252] CPU: 20 PID: 54187 Comm: ptlrpcd_02_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.320253] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.320254] task: ffff8abaee646180 ti: ffff8abaf9b9c000 task.ti: ffff8abaf9b9c000 [1488089.320257] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.320258] RSP: 0018:ffff8abaf9b9fb78 EFLAGS: 00000246 [1488089.320259] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000a10000 [1488089.320260] RDX: ffff8a8aff39b780 RSI: 0000000001810001 RDI: ffff8adafa7b1b00 [1488089.320261] RBP: ffff8abaf9b9fb78 R08: ffff8aaaffa9b780 R09: 0000000000000000 [1488089.320262] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa82d59ee00 [1488089.320263] R13: 0005c4aa17cfe700 R14: ffff8a9b7a153f00 R15: 0000000000000007 [1488089.320265] FS: 00007ea1c8885700(0000) GS:ffff8aaaffa80000(0000) knlGS:0000000000000000 [1488089.320265] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.320266] CR2: 00007ee8a005c02a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.320267] Call Trace: [1488089.320269] [] queued_spin_lock_slowpath+0xb/0xf [1488089.320271] [] _raw_spin_lock+0x20/0x30 [1488089.320279] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.320289] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.320327] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.320330] [] ? del_timer_sync+0x52/0x60 [1488089.320367] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.320404] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.320442] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.320444] [] ? wake_up_state+0x20/0x20 [1488089.320482] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.320483] [] kthread+0xd1/0xe0 [1488089.320485] [] ? insert_kthread_work+0x40/0x40 [1488089.320487] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.320489] [] ? insert_kthread_work+0x40/0x40 [1488089.320509] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.330217] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ptlrpcd_06_11:54239] [1488089.330244] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.330251] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.330254] CPU: 21 PID: 54239 Comm: ptlrpcd_06_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.330255] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.330256] task: ffff8abafbc7a080 ti: ffff8abaebaa4000 task.ti: ffff8abaebaa4000 [1488089.330260] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.330261] RSP: 0018:ffff8abaebaa7b78 EFLAGS: 00000246 [1488089.330262] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000a90000 [1488089.330263] RDX: ffff8acaffc1b780 RSI: 0000000002090001 RDI: ffff8adafa7b1b00 [1488089.330264] RBP: ffff8abaebaa7b78 R08: ffff8aeaffa9b780 R09: 0000000000000000 [1488089.330265] R10: 0000000000000000 R11: 000000000000000f R12: ffff8add0f6dfe00 [1488089.330266] R13: 0005c4aa17cdc9a0 R14: ffff8ae085aa2700 R15: 0000000000000007 [1488089.330268] FS: 00007ea1b7863700(0000) GS:ffff8aeaffa80000(0000) knlGS:0000000000000000 [1488089.330269] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.330270] CR2: 00007eab533f5000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.330271] Call Trace: [1488089.330273] [] queued_spin_lock_slowpath+0xb/0xf [1488089.330275] [] _raw_spin_lock+0x20/0x30 [1488089.330283] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.330294] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.330332] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.330370] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.330407] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.330446] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.330448] [] ? wake_up_state+0x20/0x20 [1488089.330486] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.330488] [] kthread+0xd1/0xe0 [1488089.330490] [] ? insert_kthread_work+0x40/0x40 [1488089.330492] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.330494] [] ? insert_kthread_work+0x40/0x40 [1488089.330514] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.340216] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [ptlrpcd_03_01:54193] [1488089.340244] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.340251] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.340254] CPU: 22 PID: 54193 Comm: ptlrpcd_03_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.340255] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.340256] task: ffff8abaebafd140 ti: ffff8abafd5a4000 task.ti: ffff8abafd5a4000 [1488089.340259] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.340260] RSP: 0018:ffff8abafd5a7b78 EFLAGS: 00000246 [1488089.340261] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000b10000 [1488089.340262] RDX: ffff8adaffc1b780 RSI: 0000000002190001 RDI: ffff8adafa7b1b00 [1488089.340263] RBP: ffff8abafd5a7b78 R08: ffff8abaffa9b780 R09: 0000000000000000 [1488089.340264] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aba39766000 [1488089.340265] R13: 0005c4aa17d34c00 R14: ffff8ab89e37b000 R15: 0000000000000007 [1488089.340267] FS: 00007ea1c8885700(0000) GS:ffff8abaffa80000(0000) knlGS:0000000000000000 [1488089.340268] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.340269] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.340270] Call Trace: [1488089.340272] [] queued_spin_lock_slowpath+0xb/0xf [1488089.340274] [] _raw_spin_lock+0x20/0x30 [1488089.340282] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.340293] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.340331] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.340368] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.340405] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.340443] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.340446] [] ? wake_up_state+0x20/0x20 [1488089.340484] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.340486] [] kthread+0xd1/0xe0 [1488089.340487] [] ? insert_kthread_work+0x40/0x40 [1488089.340490] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.340491] [] ? insert_kthread_work+0x40/0x40 [1488089.340511] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.348220] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ptlrpcd_07_05:54245] [1488089.348247] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.348255] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.348258] CPU: 23 PID: 54245 Comm: ptlrpcd_07_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.348259] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.348260] task: ffff8abafc521040 ti: ffff8abaf7e2c000 task.ti: ffff8abaf7e2c000 [1488089.348263] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.348264] RSP: 0018:ffff8abaf7e2fb78 EFLAGS: 00000246 [1488089.348265] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000b90000 [1488089.348266] RDX: ffff8aeaffa1b780 RSI: 0000000000290001 RDI: ffff8adafa7b1b00 [1488089.348267] RBP: ffff8abaf7e2fb78 R08: ffff8afb3f89b780 R09: 0000000000000000 [1488089.348268] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af3c4de2400 [1488089.348269] R13: 0005c4aa17ccfe10 R14: ffff8af51a43e000 R15: 0000000000000007 [1488089.348271] FS: 00007ea1c407c700(0000) GS:ffff8afb3f880000(0000) knlGS:0000000000000000 [1488089.348272] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.348273] CR2: 00007f8c23c55000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.348274] Call Trace: [1488089.348276] [] queued_spin_lock_slowpath+0xb/0xf [1488089.348278] [] _raw_spin_lock+0x20/0x30 [1488089.348286] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.348296] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.348335] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.348373] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.348410] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.348449] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.348451] [] ? wake_up_state+0x20/0x20 [1488089.348489] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.348491] [] kthread+0xd1/0xe0 [1488089.348493] [] ? insert_kthread_work+0x40/0x40 [1488089.348495] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.348497] [] ? insert_kthread_work+0x40/0x40 [1488089.348517] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.367222] NMI watchdog: BUG: soft lockup - CPU#25 stuck for 23s! [ptlrpcd_04_04:54208] [1488089.367249] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.367257] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.367259] CPU: 25 PID: 54208 Comm: ptlrpcd_04_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.367260] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.367261] task: ffff8aaba9656180 ti: ffff8abaf9210000 task.ti: ffff8abaf9210000 [1488089.367265] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.367265] RSP: 0018:ffff8abaf9213b78 EFLAGS: 00000246 [1488089.367266] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000c90000 [1488089.367268] RDX: ffff8aaaffa9b780 RSI: 0000000000a10001 RDI: ffff8adafa7b1b00 [1488089.367269] RBP: ffff8abaf9213b78 R08: ffff8acaffadb780 R09: 0000000000000000 [1488089.367269] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abddebabc00 [1488089.367270] R13: 0005c4aa17c49cd0 R14: ffff8abe5b605100 R15: 0000000000000007 [1488089.367272] FS: 00007ea1c487d700(0000) GS:ffff8acaffac0000(0000) knlGS:0000000000000000 [1488089.367273] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.367274] CR2: 00007ee0c35fc02a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.367275] Call Trace: [1488089.367277] [] queued_spin_lock_slowpath+0xb/0xf [1488089.367280] [] _raw_spin_lock+0x20/0x30 [1488089.367287] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.367298] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.367336] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.367374] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.367411] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.367450] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.367452] [] ? wake_up_state+0x20/0x20 [1488089.367491] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.367493] [] kthread+0xd1/0xe0 [1488089.367495] [] ? insert_kthread_work+0x40/0x40 [1488089.367496] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.367498] [] ? insert_kthread_work+0x40/0x40 [1488089.367518] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.375221] NMI watchdog: BUG: soft lockup - CPU#26 stuck for 23s! [ptlrpcd_01_00:54168] [1488089.375249] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.375256] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.375259] CPU: 26 PID: 54168 Comm: ptlrpcd_01_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.375260] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.375261] task: ffff8abaf6341040 ti: ffff8abaf9b5c000 task.ti: ffff8abaf9b5c000 [1488089.375265] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.375267] RSP: 0018:ffff8abaf9b5fb78 EFLAGS: 00000246 [1488089.375268] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000d10000 [1488089.375269] RDX: ffff8abaffb1b780 RSI: 0000000001310001 RDI: ffff8adafa7b1b00 [1488089.375270] RBP: ffff8abaf9b5fb78 R08: ffff8a9affadb780 R09: 0000000000000000 [1488089.375271] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8d3fec2800 [1488089.375272] R13: 0005c4aa17923260 R14: ffff8a8fb02e0000 R15: 0000000000000007 [1488089.375274] FS: 00007ea1d489d700(0000) GS:ffff8a9affac0000(0000) knlGS:0000000000000000 [1488089.375274] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.375276] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.375276] Call Trace: [1488089.375279] [] queued_spin_lock_slowpath+0xb/0xf [1488089.375281] [] _raw_spin_lock+0x20/0x30 [1488089.375289] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.375300] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.375338] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.375375] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.375413] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.375452] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.375454] [] ? wake_up_state+0x20/0x20 [1488089.375492] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.375494] [] kthread+0xd1/0xe0 [1488089.375496] [] ? insert_kthread_work+0x40/0x40 [1488089.375498] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.375500] [] ? insert_kthread_work+0x40/0x40 [1488089.375520] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.411224] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [ptlrpcd_03_06:54198] [1488089.411251] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.411258] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.411261] CPU: 30 PID: 54198 Comm: ptlrpcd_03_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.411262] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.411263] task: ffff8abaf63930c0 ti: ffff8abaf8b48000 task.ti: ffff8abaf8b48000 [1488089.411266] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.411267] RSP: 0018:ffff8abaf8b4bb78 EFLAGS: 00000246 [1488089.411268] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000f10000 [1488089.411269] RDX: ffff8a9affb1b780 RSI: 0000000001110001 RDI: ffff8adafa7b1b00 [1488089.411270] RBP: ffff8abaf8b4bb78 R08: ffff8abaffadb780 R09: 0000000000000000 [1488089.411272] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab49667aa00 [1488089.411272] R13: 0005c4aa17cc2790 R14: ffff8ab496640300 R15: 0000000000000007 [1488089.411274] FS: 00007ea1be070700(0000) GS:ffff8abaffac0000(0000) knlGS:0000000000000000 [1488089.411275] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.411277] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.411277] Call Trace: [1488089.411279] [] queued_spin_lock_slowpath+0xb/0xf [1488089.411282] [] _raw_spin_lock+0x20/0x30 [1488089.411290] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.411300] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.411338] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.411376] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.411414] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.411452] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.411455] [] ? wake_up_state+0x20/0x20 [1488089.411493] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.411495] [] kthread+0xd1/0xe0 [1488089.411497] [] ? insert_kthread_work+0x40/0x40 [1488089.411498] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.411501] [] ? insert_kthread_work+0x40/0x40 [1488089.411520] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.421224] NMI watchdog: BUG: soft lockup - CPU#31 stuck for 23s! [ptlrpcd_07_07:54247] [1488089.421253] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.421260] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.421263] CPU: 31 PID: 54247 Comm: ptlrpcd_07_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.421264] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.421265] task: ffff8abafc5230c0 ti: ffff8abaeeefc000 task.ti: ffff8abaeeefc000 [1488089.421268] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.421269] RSP: 0018:ffff8abaeeeffb78 EFLAGS: 00000246 [1488089.421270] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000f90000 [1488089.421271] RDX: ffff8a8aff45b780 RSI: 0000000002410001 RDI: ffff8adafa7b1b00 [1488089.421272] RBP: ffff8abaeeeffb78 R08: ffff8afb3f8db780 R09: 0000000000000000 [1488089.421273] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abccdbd9c00 [1488089.421274] R13: 0005c4aa17cd0290 R14: ffff8af2df98b300 R15: 0000000000000007 [1488089.421276] FS: 00007ea1b7863700(0000) GS:ffff8afb3f8c0000(0000) knlGS:0000000000000000 [1488089.421276] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.421277] CR2: 00007ec8efbd7ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.421278] Call Trace: [1488089.421280] [] queued_spin_lock_slowpath+0xb/0xf [1488089.421282] [] _raw_spin_lock+0x20/0x30 [1488089.421290] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.421301] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.421339] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.421377] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.421414] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.421453] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.421455] [] ? wake_up_state+0x20/0x20 [1488089.421493] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.421495] [] kthread+0xd1/0xe0 [1488089.421497] [] ? insert_kthread_work+0x40/0x40 [1488089.421499] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.421501] [] ? insert_kthread_work+0x40/0x40 [1488089.421521] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.431226] NMI watchdog: BUG: soft lockup - CPU#32 stuck for 23s! [ptlrpcd_00_10:54166] [1488089.431254] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.431261] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.431264] CPU: 32 PID: 54166 Comm: ptlrpcd_00_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.431265] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.431266] task: ffff8abaf815e180 ti: ffff8abafcb7c000 task.ti: ffff8abafcb7c000 [1488089.431269] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.431270] RSP: 0018:ffff8abafcb7fb78 EFLAGS: 00000246 [1488089.431271] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001010000 [1488089.431272] RDX: ffff8adaffb1b780 RSI: 0000000001190001 RDI: ffff8adafa7b1b00 [1488089.431273] RBP: ffff8abafcb7fb78 R08: ffff8a8aff31b780 R09: 0000000000000000 [1488089.431274] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a93f2d4b600 [1488089.431275] R13: 0005c4aa1878c5d0 R14: ffff8a9839221b00 R15: 0000000000000007 [1488089.431277] FS: 00007ea1c8885700(0000) GS:ffff8a8aff300000(0000) knlGS:0000000000000000 [1488089.431278] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.431279] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.431280] Call Trace: [1488089.431282] [] queued_spin_lock_slowpath+0xb/0xf [1488089.431284] [] _raw_spin_lock+0x20/0x30 [1488089.431292] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.431303] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.431340] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.431378] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.431415] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.431453] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.431455] [] ? wake_up_state+0x20/0x20 [1488089.431493] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.431495] [] kthread+0xd1/0xe0 [1488089.431497] [] ? insert_kthread_work+0x40/0x40 [1488089.431499] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.431501] [] ? insert_kthread_work+0x40/0x40 [1488089.431521] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.440226] NMI watchdog: BUG: soft lockup - CPU#33 stuck for 23s! [ptlrpcd_04_06:54210] [1488089.440254] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.440262] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.440264] CPU: 33 PID: 54210 Comm: ptlrpcd_04_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.440265] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.440267] task: ffff8abaf82c9040 ti: ffff8abaeee74000 task.ti: ffff8abaeee74000 [1488089.440270] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.440271] RSP: 0018:ffff8abaeee77b78 EFLAGS: 00000246 [1488089.440272] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001090000 [1488089.440273] RDX: ffff8afb3fa9b780 RSI: 0000000002b90001 RDI: ffff8adafa7b1b00 [1488089.440274] RBP: ffff8abaeee77b78 R08: ffff8acaffb1b780 R09: 0000000000000000 [1488089.440275] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abc1b7d1400 [1488089.440276] R13: 0005c4aa17c594d0 R14: ffff8acaa76d0c00 R15: 0000000000000007 [1488089.440277] FS: 00007ea1d68a1700(0000) GS:ffff8acaffb00000(0000) knlGS:0000000000000000 [1488089.440278] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.440280] CR2: 00007f8c27d6e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.440280] Call Trace: [1488089.440282] [] queued_spin_lock_slowpath+0xb/0xf [1488089.440284] [] _raw_spin_lock+0x20/0x30 [1488089.440293] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.440303] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.440341] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.440379] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.440416] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.440455] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.440457] [] ? wake_up_state+0x20/0x20 [1488089.440496] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.440498] [] kthread+0xd1/0xe0 [1488089.440499] [] ? insert_kthread_work+0x40/0x40 [1488089.440501] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.440503] [] ? insert_kthread_work+0x40/0x40 [1488089.440523] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.448227] NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [ptlrpcd_01_04:54172] [1488089.448255] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.448263] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.448265] CPU: 34 PID: 54172 Comm: ptlrpcd_01_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.448266] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.448267] task: ffff8abaf6345140 ti: ffff8abaf8ea4000 task.ti: ffff8abaf8ea4000 [1488089.448270] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.448271] RSP: 0018:ffff8abaf8ea7b78 EFLAGS: 00000246 [1488089.448272] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001110000 [1488089.448273] RDX: ffff8abaffbdb780 RSI: 0000000001f10000 RDI: ffff8adafa7b1b00 [1488089.448275] RBP: ffff8abaf8ea7b78 R08: ffff8a9affb1b780 R09: 0000000000000000 [1488089.448275] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8e12bde000 [1488089.448276] R13: 0005c4aa179e4750 R14: ffff8a9735b4bc00 R15: 0000000000000007 [1488089.448278] FS: 00007ea1dc8ad700(0000) GS:ffff8a9affb00000(0000) knlGS:0000000000000000 [1488089.448279] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.448280] CR2: 00007ef14000a005 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.448281] Call Trace: [1488089.448283] [] queued_spin_lock_slowpath+0xb/0xf [1488089.448285] [] _raw_spin_lock+0x20/0x30 [1488089.448293] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.448304] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.448341] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.448379] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.448416] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.448455] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.448457] [] ? wake_up_state+0x20/0x20 [1488089.448495] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.448497] [] kthread+0xd1/0xe0 [1488089.448499] [] ? insert_kthread_work+0x40/0x40 [1488089.448501] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.448503] [] ? insert_kthread_work+0x40/0x40 [1488089.448523] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.467227] NMI watchdog: BUG: soft lockup - CPU#36 stuck for 23s! [ptlrpcd_02_06:54186] [1488089.467254] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.467261] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.467263] CPU: 36 PID: 54186 Comm: ptlrpcd_02_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.467264] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.467265] task: ffff8abaee645140 ti: ffff8abaf62fc000 task.ti: ffff8abaf62fc000 [1488089.467268] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.467269] RSP: 0018:ffff8abaf62ffb78 EFLAGS: 00000246 [1488089.467270] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001210000 [1488089.467271] RDX: ffff8abaffc5b780 RSI: 0000000002710001 RDI: ffff8adafa7b1b00 [1488089.467272] RBP: ffff8abaf62ffb78 R08: ffff8aaaffb1b780 R09: 0000000000000000 [1488089.467273] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa90e867400 [1488089.467274] R13: 0005c4aa17cccae0 R14: ffff8a9dc242e600 R15: 0000000000000007 [1488089.467276] FS: 00007ea1c407c700(0000) GS:ffff8aaaffb00000(0000) knlGS:0000000000000000 [1488089.467277] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.467278] CR2: 00007f04ff0f6000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.467279] Call Trace: [1488089.467281] [] queued_spin_lock_slowpath+0xb/0xf [1488089.467283] [] _raw_spin_lock+0x20/0x30 [1488089.467291] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.467301] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.467339] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.467376] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.467413] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.467451] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.467454] [] ? wake_up_state+0x20/0x20 [1488089.467492] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.467494] [] kthread+0xd1/0xe0 [1488089.467496] [] ? insert_kthread_work+0x40/0x40 [1488089.467498] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.467499] [] ? insert_kthread_work+0x40/0x40 [1488089.467519] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.475229] NMI watchdog: BUG: soft lockup - CPU#37 stuck for 23s! [ptlrpcd_06_05:54233] [1488089.475257] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.475264] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.475267] CPU: 37 PID: 54233 Comm: ptlrpcd_06_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.475268] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.475269] task: ffff8abaf47830c0 ti: ffff8abaf6a84000 task.ti: ffff8abaf6a84000 [1488089.475272] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.475273] RSP: 0018:ffff8abaf6a87b78 EFLAGS: 00000246 [1488089.475274] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001290000 [1488089.475275] RDX: ffff8afb3fa5b780 RSI: 0000000002790001 RDI: ffff8adafa7b1b00 [1488089.475276] RBP: ffff8abaf6a87b78 R08: ffff8aeaffb1b780 R09: 0000000000000000 [1488089.475277] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ae3e3163800 [1488089.475278] R13: 0005c4aa17d361c0 R14: ffff8ae78ad31500 R15: 0000000000000007 [1488089.475280] FS: 00007ea1bb06a700(0000) GS:ffff8aeaffb00000(0000) knlGS:0000000000000000 [1488089.475281] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.475282] CR2: 00007ee0c2bf402a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.475283] Call Trace: [1488089.475285] [] queued_spin_lock_slowpath+0xb/0xf [1488089.475287] [] _raw_spin_lock+0x20/0x30 [1488089.475295] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.475306] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.475344] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.475381] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.475419] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.475458] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.475460] [] ? wake_up_state+0x20/0x20 [1488089.475498] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.475500] [] kthread+0xd1/0xe0 [1488089.475502] [] ? insert_kthread_work+0x40/0x40 [1488089.475504] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.475506] [] ? insert_kthread_work+0x40/0x40 [1488089.475526] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.484225] NMI watchdog: BUG: soft lockup - CPU#38 stuck for 23s! [ptlrpcd_03_08:54200] [1488089.484254] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.484261] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.484264] CPU: 38 PID: 54200 Comm: ptlrpcd_03_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.484265] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.484266] task: ffff8abaf6395140 ti: ffff8abaf23a8000 task.ti: ffff8abaf23a8000 [1488089.484270] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.484270] RSP: 0018:ffff8abaf23abb78 EFLAGS: 00000246 [1488089.484271] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001310000 [1488089.484272] RDX: ffff8adaffa1b780 RSI: 0000000000190001 RDI: ffff8adafa7b1b00 [1488089.484274] RBP: ffff8abaf23abb78 R08: ffff8abaffb1b780 R09: 0000000000000000 [1488089.484275] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab496651e00 [1488089.484275] R13: 0005c4aa17cc17c0 R14: ffff8aac13384500 R15: 0000000000000007 [1488089.484277] FS: 00007ea1c0074700(0000) GS:ffff8abaffb00000(0000) knlGS:0000000000000000 [1488089.484278] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.484279] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.484280] Call Trace: [1488089.484282] [] queued_spin_lock_slowpath+0xb/0xf [1488089.484284] [] _raw_spin_lock+0x20/0x30 [1488089.484292] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.484303] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.484341] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.484378] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.484415] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.484453] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.484455] [] ? wake_up_state+0x20/0x20 [1488089.484494] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.484496] [] kthread+0xd1/0xe0 [1488089.484498] [] ? insert_kthread_work+0x40/0x40 [1488089.484500] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.484502] [] ? insert_kthread_work+0x40/0x40 [1488089.484522] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.494229] NMI watchdog: BUG: soft lockup - CPU#39 stuck for 23s! [ptlrpcd_07_08:54248] [1488089.494258] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.494265] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.494268] CPU: 39 PID: 54248 Comm: ptlrpcd_07_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.494269] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.494270] task: ffff8abafc524100 ti: ffff8abaf97b8000 task.ti: ffff8abaf97b8000 [1488089.494273] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.494274] RSP: 0018:ffff8abaf97bbb78 EFLAGS: 00000246 [1488089.494275] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001390000 [1488089.494276] RDX: ffff8aaaffcdb780 RSI: 0000000002e10001 RDI: ffff8adafa7b1b00 [1488089.494277] RBP: ffff8abaf97bbb78 R08: ffff8afb3f91b780 R09: 0000000000000000 [1488089.494278] R10: 0000000000000000 R11: 000000000000000f R12: ffff8afadee70a00 [1488089.494280] R13: 0005c4aa17c56a70 R14: ffff8aef092d3900 R15: 0000000000000007 [1488089.494281] FS: 00007ea1ca889700(0000) GS:ffff8afb3f900000(0000) knlGS:0000000000000000 [1488089.494282] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.494283] CR2: 00007eea7f033bc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.494284] Call Trace: [1488089.494286] [] queued_spin_lock_slowpath+0xb/0xf [1488089.494289] [] _raw_spin_lock+0x20/0x30 [1488089.494296] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.494307] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.494345] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.494383] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.494420] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.494459] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.494461] [] ? wake_up_state+0x20/0x20 [1488089.494500] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.494502] [] kthread+0xd1/0xe0 [1488089.494504] [] ? insert_kthread_work+0x40/0x40 [1488089.494505] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.494508] [] ? insert_kthread_work+0x40/0x40 [1488089.494527] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.504227] NMI watchdog: BUG: soft lockup - CPU#40 stuck for 23s! [ptlrpcd_00_09:54165] [1488089.504255] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.504263] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.504265] CPU: 40 PID: 54165 Comm: ptlrpcd_00_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.504266] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.504268] task: ffff8abaf815d140 ti: ffff8abaf897c000 task.ti: ffff8abaf897c000 [1488089.504271] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.504272] RSP: 0018:ffff8abaf897fb78 EFLAGS: 00000246 [1488089.504273] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001410000 [1488089.504274] RDX: ffff8a8aff21b780 RSI: 0000000000010001 RDI: ffff8adafa7b1b00 [1488089.504275] RBP: ffff8abaf897fb78 R08: ffff8a8aff35b780 R09: 0000000000000000 [1488089.504276] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af7934f2400 [1488089.504277] R13: 0005c4aa1879f630 R14: ffff8a9016832700 R15: 0000000000000007 [1488089.504279] FS: 00007f1993824740(0000) GS:ffff8a8aff340000(0000) knlGS:0000000000000000 [1488089.504280] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.504281] CR2: 00007f198a91dd70 CR3: 0000005f8433a000 CR4: 00000000003407e0 [1488089.504282] Call Trace: [1488089.504284] [] queued_spin_lock_slowpath+0xb/0xf [1488089.504286] [] _raw_spin_lock+0x20/0x30 [1488089.504294] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.504305] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.504343] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.504381] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.504418] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.504457] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.504459] [] ? wake_up_state+0x20/0x20 [1488089.504498] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.504500] [] kthread+0xd1/0xe0 [1488089.504502] [] ? insert_kthread_work+0x40/0x40 [1488089.504504] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.504505] [] ? insert_kthread_work+0x40/0x40 [1488089.504526] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.521228] NMI watchdog: BUG: soft lockup - CPU#42 stuck for 22s! [ptlrpcd_01_01:54169] [1488089.521256] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.521264] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.521266] CPU: 42 PID: 54169 Comm: ptlrpcd_01_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.521267] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.521268] task: ffff8abaf6342080 ti: ffff8abaf1700000 task.ti: ffff8abaf1700000 [1488089.521272] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.521273] RSP: 0018:ffff8abaf1703b78 EFLAGS: 00000246 [1488089.521274] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001510000 [1488089.521274] RDX: ffff8aaaffb9b780 RSI: 0000000001a10001 RDI: ffff8adafa7b1b00 [1488089.521276] RBP: ffff8abaf1703b78 R08: ffff8a9affb5b780 R09: 0000000000000000 [1488089.521277] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a94d1231600 [1488089.521277] R13: 0005c4aa179bd630 R14: ffff8a8ed6a73c00 R15: 0000000000000007 [1488089.521279] FS: 00007ea1b2859700(0000) GS:ffff8a9affb40000(0000) knlGS:0000000000000000 [1488089.521280] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.521281] CR2: 00007ef4a7d2d000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.521282] Call Trace: [1488089.521284] [] queued_spin_lock_slowpath+0xb/0xf [1488089.521286] [] _raw_spin_lock+0x20/0x30 [1488089.521294] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.521305] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.521343] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.521381] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.521419] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.521457] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.521459] [] ? wake_up_state+0x20/0x20 [1488089.521498] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.521500] [] kthread+0xd1/0xe0 [1488089.521501] [] ? insert_kthread_work+0x40/0x40 [1488089.521504] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.521506] [] ? insert_kthread_work+0x40/0x40 [1488089.521526] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.531231] NMI watchdog: BUG: soft lockup - CPU#43 stuck for 22s! [ptlrpcd_05_10:54226] [1488089.531259] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.531266] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.531269] CPU: 43 PID: 54226 Comm: ptlrpcd_05_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.531270] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.531272] task: ffff8abafb33b0c0 ti: ffff8abaeeef4000 task.ti: ffff8abaeeef4000 [1488089.531276] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.531276] RSP: 0018:ffff8abaeeef7b78 EFLAGS: 00000246 [1488089.531277] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001590000 [1488089.531279] RDX: ffff8afb3f8db780 RSI: 0000000000f90001 RDI: ffff8adafa7b1b00 [1488089.531280] RBP: ffff8abaeeef7b78 R08: ffff8adaffb5b780 R09: 0000000000000000 [1488089.531280] R10: 0000000000000000 R11: 000000000000000f R12: ffff8acb4b436800 [1488089.531282] R13: 0005c4aa1785f220 R14: ffff8a9421aa0900 R15: 0000000000000007 [1488089.531283] FS: 00007ea1b2859700(0000) GS:ffff8adaffb40000(0000) knlGS:0000000000000000 [1488089.531284] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.531285] CR2: 0000000002908888 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.531286] Call Trace: [1488089.531289] [] queued_spin_lock_slowpath+0xb/0xf [1488089.531291] [] _raw_spin_lock+0x20/0x30 [1488089.531299] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.531309] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.531348] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.531386] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.531423] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.531461] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.531464] [] ? wake_up_state+0x20/0x20 [1488089.531502] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.531504] [] kthread+0xd1/0xe0 [1488089.531506] [] ? insert_kthread_work+0x40/0x40 [1488089.531508] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.531510] [] ? insert_kthread_work+0x40/0x40 [1488089.531530] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.541234] NMI watchdog: BUG: soft lockup - CPU#44 stuck for 22s! [ptlrpcd_02_08:54188] [1488089.541262] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.541269] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.541272] CPU: 44 PID: 54188 Comm: ptlrpcd_02_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.541273] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.541274] task: ffff8abaebaf8000 ti: ffff8abaecba8000 task.ti: ffff8abaecba8000 [1488089.541277] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.541278] RSP: 0018:ffff8abaecbabb78 EFLAGS: 00000246 [1488089.541279] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001610000 [1488089.541280] RDX: ffff8aeaffc1b780 RSI: 0000000002290001 RDI: ffff8adafa7b1b00 [1488089.541281] RBP: ffff8abaecbabb78 R08: ffff8aaaffb5b780 R09: 0000000000000000 [1488089.541283] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9b52033000 [1488089.541284] R13: 0005c4aa17cd9280 R14: ffff8aa69eaf9800 R15: 0000000000000007 [1488089.541285] FS: 00007ea1ba068700(0000) GS:ffff8aaaffb40000(0000) knlGS:0000000000000000 [1488089.541286] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.541287] CR2: 00007edc08f4ef58 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.541288] Call Trace: [1488089.541290] [] queued_spin_lock_slowpath+0xb/0xf [1488089.541292] [] _raw_spin_lock+0x20/0x30 [1488089.541300] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.541311] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.541349] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.541386] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.541423] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.541462] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.541464] [] ? wake_up_state+0x20/0x20 [1488089.541502] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.541504] [] kthread+0xd1/0xe0 [1488089.541506] [] ? insert_kthread_work+0x40/0x40 [1488089.541508] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.541509] [] ? insert_kthread_work+0x40/0x40 [1488089.541529] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.558233] NMI watchdog: BUG: soft lockup - CPU#46 stuck for 22s! [ptlrpcd_03_07:54199] [1488089.558259] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.558267] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.558269] CPU: 46 PID: 54199 Comm: ptlrpcd_03_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.558270] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.558271] task: ffff8abaf6394100 ti: ffff8abafbfc0000 task.ti: ffff8abafbfc0000 [1488089.558274] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.558276] RSP: 0018:ffff8abafbfc3b78 EFLAGS: 00000246 [1488089.558277] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001710000 [1488089.558277] RDX: ffff8afb3f85b780 RSI: 0000000000790001 RDI: ffff8adafa7b1b00 [1488089.558279] RBP: ffff8abafbfc3b78 R08: ffff8abaffb5b780 R09: 0000000000000000 [1488089.558280] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aabdc308400 [1488089.558280] R13: 0005c4aa17d34e70 R14: ffff8ab441780900 R15: 0000000000000007 [1488089.558282] FS: 00007f1993824740(0000) GS:ffff8abaffb40000(0000) knlGS:0000000000000000 [1488089.558283] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.558285] CR2: 00007f002a73a795 CR3: 0000005f8433a000 CR4: 00000000003407e0 [1488089.558285] Call Trace: [1488089.558287] [] queued_spin_lock_slowpath+0xb/0xf [1488089.558290] [] _raw_spin_lock+0x20/0x30 [1488089.558298] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.558308] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.558346] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.558384] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.558421] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.558460] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.558462] [] ? wake_up_state+0x20/0x20 [1488089.558501] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.558502] [] kthread+0xd1/0xe0 [1488089.558504] [] ? insert_kthread_work+0x40/0x40 [1488089.558506] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.558509] [] ? insert_kthread_work+0x40/0x40 [1488089.558529] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.568230] NMI watchdog: BUG: soft lockup - CPU#47 stuck for 22s! [ptlrpcd_07_02:54242] [1488089.568256] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.568264] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.568267] CPU: 47 PID: 54242 Comm: ptlrpcd_07_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.568268] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.568269] task: ffff8abafbc7d140 ti: ffff8abafcdb4000 task.ti: ffff8abafcdb4000 [1488089.568272] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.568273] RSP: 0018:ffff8abafcdb7b78 EFLAGS: 00000246 [1488089.568274] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001790000 [1488089.568275] RDX: ffff8abaffa1b780 RSI: 0000000000310001 RDI: ffff8adafa7b1b00 [1488089.568276] RBP: ffff8abafcdb7b78 R08: ffff8afb3f95b780 R09: 0000000000000000 [1488089.568277] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af4f0abdc00 [1488089.568278] R13: 0005c4aa17bf9cb0 R14: ffff8af5fcf29e00 R15: 0000000000000007 [1488089.568280] FS: 00007ea1c9887700(0000) GS:ffff8afb3f940000(0000) knlGS:0000000000000000 [1488089.568281] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.568282] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.568283] Call Trace: [1488089.568285] [] queued_spin_lock_slowpath+0xb/0xf [1488089.568287] [] _raw_spin_lock+0x20/0x30 [1488089.568295] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.568306] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.568344] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.568382] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.568419] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.568458] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.568460] [] ? wake_up_state+0x20/0x20 [1488089.568499] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.568501] [] kthread+0xd1/0xe0 [1488089.568503] [] ? insert_kthread_work+0x40/0x40 [1488089.568504] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.568507] [] ? insert_kthread_work+0x40/0x40 [1488089.568526] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.586234] NMI watchdog: BUG: soft lockup - CPU#49 stuck for 22s! [ptlrpcd_04_08:54212] [1488089.586263] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.586270] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.586273] CPU: 49 PID: 54212 Comm: ptlrpcd_04_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.586273] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.586275] task: ffff8abaf82cb0c0 ti: ffff8abaedea0000 task.ti: ffff8abaedea0000 [1488089.586278] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.586279] RSP: 0018:ffff8abaedea3b78 EFLAGS: 00000246 [1488089.586280] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001890000 [1488089.586281] RDX: ffff8afb3f95b780 RSI: 0000000001790001 RDI: ffff8adafa7b1b00 [1488089.586282] RBP: ffff8abaedea3b78 R08: ffff8acaffb9b780 R09: 0000000000000000 [1488089.586283] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abf38b6d000 [1488089.586284] R13: 0005c4aa17c5ad80 R14: ffff8acae8391200 R15: 0000000000000007 [1488089.586286] FS: 00007ea1de0b0700(0000) GS:ffff8acaffb80000(0000) knlGS:0000000000000000 [1488089.586287] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.586288] CR2: 00007ec111dde00a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.586289] Call Trace: [1488089.586291] [] queued_spin_lock_slowpath+0xb/0xf [1488089.586293] [] _raw_spin_lock+0x20/0x30 [1488089.586301] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.586312] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.586350] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.586388] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.586425] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.586464] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.586466] [] ? wake_up_state+0x20/0x20 [1488089.586504] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.586506] [] kthread+0xd1/0xe0 [1488089.586508] [] ? insert_kthread_work+0x40/0x40 [1488089.586510] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.586512] [] ? insert_kthread_work+0x40/0x40 [1488089.586532] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.613237] NMI watchdog: BUG: soft lockup - CPU#52 stuck for 22s! [ptlrpcd_02_09:54189] [1488089.613264] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.613272] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.613274] CPU: 52 PID: 54189 Comm: ptlrpcd_02_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.613275] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.613276] task: ffff8abaebaf9040 ti: ffff8abaf7e44000 task.ti: ffff8abaf7e44000 [1488089.613279] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.613280] RSP: 0018:ffff8abaf7e47b78 EFLAGS: 00000246 [1488089.613281] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001a10000 [1488089.613282] RDX: ffff8aaaffa1b780 RSI: 0000000000210001 RDI: ffff8adafa7b1b00 [1488089.613283] RBP: ffff8abaf7e47b78 R08: ffff8aaaffb9b780 R09: 0000000000000000 [1488089.613284] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa83ad31e00 [1488089.613285] R13: 0005c4aa17cfec30 R14: ffff8a9bcba58f00 R15: 0000000000000007 [1488089.613287] FS: 00007ea1d509e700(0000) GS:ffff8aaaffb80000(0000) knlGS:0000000000000000 [1488089.613288] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.613289] CR2: 00007eea7f02bbc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.613290] Call Trace: [1488089.613293] [] queued_spin_lock_slowpath+0xb/0xf [1488089.613295] [] _raw_spin_lock+0x20/0x30 [1488089.613303] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.613314] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.613351] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.613389] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.613425] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.613464] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.613466] [] ? wake_up_state+0x20/0x20 [1488089.613504] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.613505] [] kthread+0xd1/0xe0 [1488089.613507] [] ? insert_kthread_work+0x40/0x40 [1488089.613509] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.613511] [] ? insert_kthread_work+0x40/0x40 [1488089.613531] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.630234] NMI watchdog: BUG: soft lockup - CPU#54 stuck for 22s! [ptlrpcd_03_04:54196] [1488089.630261] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.630269] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.630271] CPU: 54 PID: 54196 Comm: ptlrpcd_03_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.630272] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.630273] task: ffff8abaf6391040 ti: ffff8aba98688000 task.ti: ffff8aba98688000 [1488089.630276] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.630277] RSP: 0018:ffff8aba9868bb78 EFLAGS: 00000246 [1488089.630278] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001b10000 [1488089.630279] RDX: ffff8aeaffc9b780 RSI: 0000000002a90001 RDI: ffff8adafa7b1b00 [1488089.630280] RBP: ffff8aba9868bb78 R08: ffff8abaffb9b780 R09: 0000000000000000 [1488089.630281] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab8d9dd2e00 [1488089.630282] R13: 0005c4aa17cb5c70 R14: ffff8ab98a500c00 R15: 0000000000000007 [1488089.630284] FS: 00007ea1c7082700(0000) GS:ffff8abaffb80000(0000) knlGS:0000000000000000 [1488089.630285] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.630286] CR2: 00007f3dc9661978 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.630287] Call Trace: [1488089.630289] [] queued_spin_lock_slowpath+0xb/0xf [1488089.630290] [] _raw_spin_lock+0x20/0x30 [1488089.630298] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.630309] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.630346] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.630384] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.630421] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.630459] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.630461] [] ? wake_up_state+0x20/0x20 [1488089.630499] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.630501] [] kthread+0xd1/0xe0 [1488089.630503] [] ? insert_kthread_work+0x40/0x40 [1488089.630505] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.630507] [] ? insert_kthread_work+0x40/0x40 [1488089.630527] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.639236] NMI watchdog: BUG: soft lockup - CPU#55 stuck for 22s! [ptlrpcd_07_09:54249] [1488089.639265] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.639272] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.639275] CPU: 55 PID: 54249 Comm: ptlrpcd_07_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.639276] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.639277] task: ffff8abafc525140 ti: ffff8abaee6c0000 task.ti: ffff8abaee6c0000 [1488089.639280] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.639281] RSP: 0018:ffff8abaee6c3b78 EFLAGS: 00000246 [1488089.639282] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001b90000 [1488089.639283] RDX: ffff8aeaffbdb780 RSI: 0000000001e90001 RDI: ffff8adafa7b1b00 [1488089.639284] RBP: ffff8abaee6c3b78 R08: ffff8afb3f99b780 R09: 0000000000000000 [1488089.639285] R10: 0000000000000000 R11: 000000000000000f R12: ffff8afaaa72c600 [1488089.639286] R13: 0005c4aa17cce010 R14: ffff8af51a438000 R15: 0000000000000007 [1488089.639288] FS: 00007ea1cf893700(0000) GS:ffff8afb3f980000(0000) knlGS:0000000000000000 [1488089.639289] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.639290] CR2: 00007f6e7e5bf590 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.639290] Call Trace: [1488089.639293] [] queued_spin_lock_slowpath+0xb/0xf [1488089.639295] [] _raw_spin_lock+0x20/0x30 [1488089.639303] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.639313] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.639351] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.639389] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.639426] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.639464] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.639467] [] ? wake_up_state+0x20/0x20 [1488089.639505] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.639507] [] kthread+0xd1/0xe0 [1488089.639509] [] ? insert_kthread_work+0x40/0x40 [1488089.639511] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.639513] [] ? insert_kthread_work+0x40/0x40 [1488089.639533] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.657239] NMI watchdog: BUG: soft lockup - CPU#57 stuck for 22s! [ptlrpcd_04_09:54213] [1488089.657265] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.657272] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.657275] CPU: 57 PID: 54213 Comm: ptlrpcd_04_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.657276] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.657277] task: ffff8abaf82cc100 ti: ffff8abaf69e0000 task.ti: ffff8abaf69e0000 [1488089.657280] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.657281] RSP: 0018:ffff8abaf69e3b78 EFLAGS: 00000246 [1488089.657282] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001c90000 [1488089.657283] RDX: ffff8acaffb5b780 RSI: 0000000001490001 RDI: ffff8adafa7b1b00 [1488089.657284] RBP: ffff8abaf69e3b78 R08: ffff8acaffbdb780 R09: 0000000000000000 [1488089.657285] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac421387200 [1488089.657286] R13: 0005c4aa17cc5280 R14: ffff8ac42130c800 R15: 0000000000000007 [1488089.657288] FS: 00007ea1bb06a700(0000) GS:ffff8acaffbc0000(0000) knlGS:0000000000000000 [1488089.657289] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.657290] CR2: 00007f8c23c55000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.657291] Call Trace: [1488089.657293] [] queued_spin_lock_slowpath+0xb/0xf [1488089.657295] [] _raw_spin_lock+0x20/0x30 [1488089.657303] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.657313] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.657351] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.657353] [] ? del_timer_sync+0x52/0x60 [1488089.657390] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.657427] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.657466] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.657468] [] ? wake_up_state+0x20/0x20 [1488089.657507] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.657508] [] kthread+0xd1/0xe0 [1488089.657510] [] ? insert_kthread_work+0x40/0x40 [1488089.657512] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.657514] [] ? insert_kthread_work+0x40/0x40 [1488089.657534] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.686237] NMI watchdog: BUG: soft lockup - CPU#60 stuck for 22s! [ptlrpcd_02_04:54184] [1488089.686263] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.686270] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.686272] CPU: 60 PID: 54184 Comm: ptlrpcd_02_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.686273] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.686274] task: ffff8abaee6430c0 ti: ffff8abaed324000 task.ti: ffff8abaed324000 [1488089.686277] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.686278] RSP: 0018:ffff8abaed327b78 EFLAGS: 00000246 [1488089.686279] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001e10000 [1488089.686280] RDX: ffff8aeaffc5b780 RSI: 0000000002690001 RDI: ffff8adafa7b1b00 [1488089.686282] RBP: ffff8abaed327b78 R08: ffff8aaaffbdb780 R09: 0000000000000000 [1488089.686282] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa8abf4ac00 [1488089.686283] R13: 0005c4aa17cdb290 R14: ffff8a9c8c644b00 R15: 0000000000000007 [1488089.686285] FS: 00007ea1d1897700(0000) GS:ffff8aaaffbc0000(0000) knlGS:0000000000000000 [1488089.686286] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.686287] CR2: 00007f8c1846c000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.686288] Call Trace: [1488089.686290] [] queued_spin_lock_slowpath+0xb/0xf [1488089.686292] [] _raw_spin_lock+0x20/0x30 [1488089.686300] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.686310] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.686348] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.686386] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.686423] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.686461] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.686463] [] ? wake_up_state+0x20/0x20 [1488089.686502] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.686504] [] kthread+0xd1/0xe0 [1488089.686505] [] ? insert_kthread_work+0x40/0x40 [1488089.686507] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.686509] [] ? insert_kthread_work+0x40/0x40 [1488089.686529] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.703243] NMI watchdog: BUG: soft lockup - CPU#62 stuck for 22s! [ptlrpcd_03_09:54201] [1488089.703269] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.703277] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.703279] CPU: 62 PID: 54201 Comm: ptlrpcd_03_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.703280] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.703281] task: ffff8abaf6396180 ti: ffff8abaf8fd0000 task.ti: ffff8abaf8fd0000 [1488089.703284] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.703285] RSP: 0018:ffff8abaf8fd3b78 EFLAGS: 00000246 [1488089.703286] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001f10000 [1488089.703287] RDX: ffff8abaffc9b780 RSI: 0000000002b10001 RDI: ffff8adafa7b1b00 [1488089.703288] RBP: ffff8abaf8fd3b78 R08: ffff8abaffbdb780 R09: 0000000000000000 [1488089.703289] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab759a35400 [1488089.703290] R13: 0005c4aa17d30930 R14: ffff8ab7c3442100 R15: 0000000000000007 [1488089.703292] FS: 00007ea1dc0ac700(0000) GS:ffff8abaffbc0000(0000) knlGS:0000000000000000 [1488089.703293] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.703294] CR2: 00007f8c2c495000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.703295] Call Trace: [1488089.703297] [] queued_spin_lock_slowpath+0xb/0xf [1488089.703300] [] _raw_spin_lock+0x20/0x30 [1488089.703307] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.703318] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.703356] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.703394] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.703431] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.703469] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.703472] [] ? wake_up_state+0x20/0x20 [1488089.703510] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.703512] [] kthread+0xd1/0xe0 [1488089.703514] [] ? insert_kthread_work+0x40/0x40 [1488089.703516] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.703518] [] ? insert_kthread_work+0x40/0x40 [1488089.703538] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.723244] NMI watchdog: BUG: soft lockup - CPU#64 stuck for 22s! [ptlrpcd_00_06:54162] [1488089.723271] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.723279] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.723281] CPU: 64 PID: 54162 Comm: ptlrpcd_00_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.723282] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.723283] task: ffff8abaf815a080 ti: ffff8abafcf18000 task.ti: ffff8abafcf18000 [1488089.723286] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.723287] RSP: 0018:ffff8abafcf1bb78 EFLAGS: 00000246 [1488089.723288] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002010000 [1488089.723289] RDX: ffff8a9affa1b780 RSI: 0000000000110001 RDI: ffff8adafa7b1b00 [1488089.723290] RBP: ffff8abafcf1bb78 R08: ffff8a8aff41b780 R09: 0000000000000000 [1488089.723291] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c3200 [1488089.723292] R13: 0005c4aa1879ffe0 R14: ffff8a90d9b17500 R15: 0000000000000007 [1488089.723294] FS: 00007f471ea2d700(0000) GS:ffff8a8aff400000(0000) knlGS:0000000000000000 [1488089.723295] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.723296] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.723297] Call Trace: [1488089.723299] [] queued_spin_lock_slowpath+0xb/0xf [1488089.723301] [] _raw_spin_lock+0x20/0x30 [1488089.723309] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.723319] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.723357] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.723395] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.723431] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.723470] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.723472] [] ? wake_up_state+0x20/0x20 [1488089.723510] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.723512] [] kthread+0xd1/0xe0 [1488089.723514] [] ? insert_kthread_work+0x40/0x40 [1488089.723516] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.723518] [] ? insert_kthread_work+0x40/0x40 [1488089.723538] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.732245] NMI watchdog: BUG: soft lockup - CPU#65 stuck for 22s! [ptlrpcd_04_02:54206] [1488089.732271] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.732279] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.732281] CPU: 65 PID: 54206 Comm: ptlrpcd_04_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.732282] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.732283] task: ffff8aaba9654100 ti: ffff8abaed798000 task.ti: ffff8abaed798000 [1488089.732286] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.732287] RSP: 0018:ffff8abaed79bb78 EFLAGS: 00000246 [1488089.732288] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002090000 [1488089.732289] RDX: ffff8adaffc5b780 RSI: 0000000002590001 RDI: ffff8adafa7b1b00 [1488089.732290] RBP: ffff8abaed79bb78 R08: ffff8acaffc1b780 R09: 0000000000000000 [1488089.732291] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abf71fb9a00 [1488089.732292] R13: 0005c4aa17c537a0 R14: ffff8abe5b6fe300 R15: 0000000000000007 [1488089.732294] FS: 00007ea1be871700(0000) GS:ffff8acaffc00000(0000) knlGS:0000000000000000 [1488089.732294] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.732295] CR2: 00007ee8a0ba6013 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.732297] Call Trace: [1488089.732299] [] queued_spin_lock_slowpath+0xb/0xf [1488089.732301] [] _raw_spin_lock+0x20/0x30 [1488089.732309] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.732319] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.732357] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.732395] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.732432] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.732470] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.732473] [] ? wake_up_state+0x20/0x20 [1488089.732511] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.732513] [] kthread+0xd1/0xe0 [1488089.732515] [] ? insert_kthread_work+0x40/0x40 [1488089.732517] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.732519] [] ? insert_kthread_work+0x40/0x40 [1488089.732539] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.740241] NMI watchdog: BUG: soft lockup - CPU#66 stuck for 22s! [ptlrpcd_01_11:54179] [1488089.740268] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.740275] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.740278] CPU: 66 PID: 54179 Comm: ptlrpcd_01_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.740279] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.740280] task: ffff8abaf4dad140 ti: ffff8abaf8720000 task.ti: ffff8abaf8720000 [1488089.740283] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.740284] RSP: 0018:ffff8abaf8723b78 EFLAGS: 00000246 [1488089.740285] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002110000 [1488089.740287] RDX: ffff8a9affa5b780 RSI: 0000000000510001 RDI: ffff8adafa7b1b00 [1488089.740288] RBP: ffff8abaf8723b78 R08: ffff8a9affc1b780 R09: 0000000000000000 [1488089.740288] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a90f4378600 [1488089.740290] R13: 0005c4aa17a3c990 R14: ffff8a931021e900 R15: 0000000000000007 [1488089.740291] FS: 00007ea1cb88b700(0000) GS:ffff8a9affc00000(0000) knlGS:0000000000000000 [1488089.740292] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.740293] CR2: 00007ed1c5b95005 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.740294] Call Trace: [1488089.740296] [] queued_spin_lock_slowpath+0xb/0xf [1488089.740298] [] _raw_spin_lock+0x20/0x30 [1488089.740306] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.740317] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.740355] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.740393] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.740430] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.740468] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.740471] [] ? wake_up_state+0x20/0x20 [1488089.740509] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.740511] [] kthread+0xd1/0xe0 [1488089.740513] [] ? insert_kthread_work+0x40/0x40 [1488089.740515] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.740517] [] ? insert_kthread_work+0x40/0x40 [1488089.740537] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.768246] NMI watchdog: BUG: soft lockup - CPU#69 stuck for 22s! [ptlrpcd_06_06:54234] [1488089.768274] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.768282] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.768285] CPU: 69 PID: 54234 Comm: ptlrpcd_06_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.768285] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.768287] task: ffff8abaf4784100 ti: ffff8abaefaf4000 task.ti: ffff8abaefaf4000 [1488089.768290] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.768291] RSP: 0018:ffff8abaefaf7b78 EFLAGS: 00000246 [1488089.768292] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002290000 [1488089.768293] RDX: ffff8a9affb9b780 RSI: 0000000001910001 RDI: ffff8adafa7b1b00 [1488089.768293] RBP: ffff8abaefaf7b78 R08: ffff8aeaffc1b780 R09: 0000000000000000 [1488089.768295] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adccf56cc00 [1488089.768296] R13: 0005c4aa17c91e00 R14: ffff8aeaaa641200 R15: 0000000000000007 [1488089.768297] FS: 00007ea1b7863700(0000) GS:ffff8aeaffc00000(0000) knlGS:0000000000000000 [1488089.768298] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.768299] CR2: 00007f47239c9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.768300] Call Trace: [1488089.768301] [] queued_spin_lock_slowpath+0xb/0xf [1488089.768303] [] _raw_spin_lock+0x20/0x30 [1488089.768311] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.768321] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.768359] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.768396] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.768433] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.768471] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.768473] [] ? wake_up_state+0x20/0x20 [1488089.768511] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.768513] [] kthread+0xd1/0xe0 [1488089.768514] [] ? insert_kthread_work+0x40/0x40 [1488089.768516] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.768518] [] ? insert_kthread_work+0x40/0x40 [1488089.768538] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.777247] NMI watchdog: BUG: soft lockup - CPU#70 stuck for 22s! [ptlrpcd_03_03:54195] [1488089.777274] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.777282] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.777284] CPU: 70 PID: 54195 Comm: ptlrpcd_03_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.777285] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.777286] task: ffff8abaf6390000 ti: ffff8abaf86a0000 task.ti: ffff8abaf86a0000 [1488089.777289] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.777290] RSP: 0018:ffff8abaf86a3b78 EFLAGS: 00000246 [1488089.777291] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002310000 [1488089.777291] RDX: ffff8a8aff3db780 RSI: 0000000001c10001 RDI: ffff8adafa7b1b00 [1488089.777293] RBP: ffff8abaf86a3b78 R08: ffff8abaffc1b780 R09: 0000000000000000 [1488089.777294] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abaf1e96a00 [1488089.777295] R13: 0005c4aa17d25530 R14: ffff8aafa8556000 R15: 0000000000000007 [1488089.777297] FS: 00007ea1c7082700(0000) GS:ffff8abaffc00000(0000) knlGS:0000000000000000 [1488089.777297] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.777299] CR2: 00007ebde3c2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.777300] Call Trace: [1488089.777302] [] queued_spin_lock_slowpath+0xb/0xf [1488089.777304] [] _raw_spin_lock+0x20/0x30 [1488089.777312] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.777322] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.777360] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.777398] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.777434] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.777473] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.777475] [] ? wake_up_state+0x20/0x20 [1488089.777513] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.777514] [] kthread+0xd1/0xe0 [1488089.777516] [] ? insert_kthread_work+0x40/0x40 [1488089.777518] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.777520] [] ? insert_kthread_work+0x40/0x40 [1488089.777540] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.787249] NMI watchdog: BUG: soft lockup - CPU#71 stuck for 22s! [ptlrpcd_07_11:54251] [1488089.787275] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.787282] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.787285] CPU: 71 PID: 54251 Comm: ptlrpcd_07_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.787286] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.787287] task: ffff8abafc7d8000 ti: ffff8abaf53fc000 task.ti: ffff8abaf53fc000 [1488089.787291] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x128/0x200 [1488089.787291] RSP: 0018:ffff8abaf53ffb78 EFLAGS: 00000246 [1488089.787292] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002390000 [1488089.787294] RDX: ffff8acaffb1b780 RSI: 0000000001090001 RDI: ffff8adafa7b1b00 [1488089.787295] RBP: ffff8abaf53ffb78 R08: ffff8afb3fa1b780 R09: 0000000000000000 [1488089.787295] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af57d7fd600 [1488089.787297] R13: 0005c4aa17cd7110 R14: ffff8aec26dd7b00 R15: 0000000000000007 [1488089.787298] FS: 00007ea1d389b700(0000) GS:ffff8afb3fa00000(0000) knlGS:0000000000000000 [1488089.787299] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.787300] CR2: 00007ec111bda000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.787301] Call Trace: [1488089.787303] [] queued_spin_lock_slowpath+0xb/0xf [1488089.787305] [] _raw_spin_lock+0x20/0x30 [1488089.787313] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.787324] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.787362] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.787400] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.787436] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.787475] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.787478] [] ? wake_up_state+0x20/0x20 [1488089.787516] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.787518] [] kthread+0xd1/0xe0 [1488089.787520] [] ? insert_kthread_work+0x40/0x40 [1488089.787521] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.787524] [] ? insert_kthread_work+0x40/0x40 [1488089.787544] Code: 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 <74> f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 [1488089.795245] NMI watchdog: BUG: soft lockup - CPU#72 stuck for 22s! [ptlrpcd_00_04:54160] [1488089.795272] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.795280] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.795282] CPU: 72 PID: 54160 Comm: ptlrpcd_00_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.795283] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.795285] task: ffff8abaf8158000 ti: ffff8abaf4a70000 task.ti: ffff8abaf4a70000 [1488089.795288] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.795289] RSP: 0018:ffff8abaf4a73b78 EFLAGS: 00000246 [1488089.795290] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002410000 [1488089.795291] RDX: ffff8aeaffadb780 RSI: 0000000000e90001 RDI: ffff8adafa7b1b00 [1488089.795292] RBP: ffff8abaf4a73b78 R08: ffff8a8aff45b780 R09: 0000000000000000 [1488089.795293] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c0200 [1488089.795294] R13: 0005c4aa1879fac0 R14: ffff8a7b02ec6f00 R15: 0000000000000007 [1488089.795296] FS: 00007ea1ce090700(0000) GS:ffff8a8aff440000(0000) knlGS:0000000000000000 [1488089.795296] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.795298] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.795298] Call Trace: [1488089.795301] [] queued_spin_lock_slowpath+0xb/0xf [1488089.795303] [] _raw_spin_lock+0x20/0x30 [1488089.795311] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.795322] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.795359] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.795397] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.795434] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.795472] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.795474] [] ? wake_up_state+0x20/0x20 [1488089.795512] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.795514] [] kthread+0xd1/0xe0 [1488089.795516] [] ? insert_kthread_work+0x40/0x40 [1488089.795518] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.795520] [] ? insert_kthread_work+0x40/0x40 [1488089.795540] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.804251] NMI watchdog: BUG: soft lockup - CPU#73 stuck for 22s! [ptlrpcd_04_01:54205] [1488089.804278] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.804286] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.804289] CPU: 73 PID: 54205 Comm: ptlrpcd_04_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.804289] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.804291] task: ffff8aaba96530c0 ti: ffff8abaf9410000 task.ti: ffff8abaf9410000 [1488089.804294] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.804295] RSP: 0018:ffff8abaf9413b78 EFLAGS: 00000246 [1488089.804296] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002490000 [1488089.804296] RDX: ffff8abaffa9b780 RSI: 0000000000b10001 RDI: ffff8adafa7b1b00 [1488089.804298] RBP: ffff8abaf9413b78 R08: ffff8acaffc5b780 R09: 0000000000000000 [1488089.804299] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aca6477ea00 [1488089.804299] R13: 0005c4aa17cb5950 R14: ffff8acaace2c500 R15: 0000000000000007 [1488089.804301] FS: 00007ea1c8084700(0000) GS:ffff8acaffc40000(0000) knlGS:0000000000000000 [1488089.804302] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.804303] CR2: 00007fab9fc94ac0 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.804304] Call Trace: [1488089.804306] [] queued_spin_lock_slowpath+0xb/0xf [1488089.804308] [] _raw_spin_lock+0x20/0x30 [1488089.804316] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.804326] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.804364] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.804402] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.804439] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.804478] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.804480] [] ? wake_up_state+0x20/0x20 [1488089.804518] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.804520] [] kthread+0xd1/0xe0 [1488089.804522] [] ? insert_kthread_work+0x40/0x40 [1488089.804524] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.804526] [] ? insert_kthread_work+0x40/0x40 [1488089.804546] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.823251] NMI watchdog: BUG: soft lockup - CPU#75 stuck for 22s! [ptlrpcd_05_11:54227] [1488089.823278] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.823285] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.823288] CPU: 75 PID: 54227 Comm: ptlrpcd_05_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.823289] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.823290] task: ffff8abafb33c100 ti: ffff8abafc494000 task.ti: ffff8abafc494000 [1488089.823293] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.823294] RSP: 0018:ffff8abafc497b78 EFLAGS: 00000246 [1488089.823295] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002590000 [1488089.823296] RDX: ffff8adaffa9b780 RSI: 0000000000990001 RDI: ffff8adafa7b1b00 [1488089.823297] RBP: ffff8abafc497b78 R08: ffff8adaffc5b780 R09: 0000000000000000 [1488089.823297] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad07cfce000 [1488089.823298] R13: 0005c4aa179b3300 R14: ffff8ada99cc5d00 R15: 0000000000000007 [1488089.823300] FS: 00007ea1b2859700(0000) GS:ffff8adaffc40000(0000) knlGS:0000000000000000 [1488089.823301] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.823302] CR2: 00007f8c11e1b000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.823303] Call Trace: [1488089.823305] [] queued_spin_lock_slowpath+0xb/0xf [1488089.823307] [] _raw_spin_lock+0x20/0x30 [1488089.823315] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.823326] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.823364] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.823402] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.823439] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.823477] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.823480] [] ? wake_up_state+0x20/0x20 [1488089.823518] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.823520] [] kthread+0xd1/0xe0 [1488089.823522] [] ? insert_kthread_work+0x40/0x40 [1488089.823523] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.823526] [] ? insert_kthread_work+0x40/0x40 [1488089.823545] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.831251] NMI watchdog: BUG: soft lockup - CPU#76 stuck for 22s! [ptlrpcd_02_11:54191] [1488089.831278] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.831286] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.831288] CPU: 76 PID: 54191 Comm: ptlrpcd_02_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.831289] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.831290] task: ffff8abaebafb0c0 ti: ffff8abaeef54000 task.ti: ffff8abaeef54000 [1488089.831294] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.831295] RSP: 0018:ffff8abaeef57b78 EFLAGS: 00000246 [1488089.831296] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002610000 [1488089.831297] RDX: ffff8acaffa1b780 RSI: 0000000000090001 RDI: ffff8adafa7b1b00 [1488089.831297] RBP: ffff8abaeef57b78 R08: ffff8aaaffc5b780 R09: 0000000000000000 [1488089.831299] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa690382600 [1488089.831300] R13: 0005c4aa17cfa010 R14: ffff8adf00a89e00 R15: 0000000000000007 [1488089.831301] FS: 00007ea1b585f700(0000) GS:ffff8aaaffc40000(0000) knlGS:0000000000000000 [1488089.831302] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.831303] CR2: 00007f8c1d455000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.831304] Call Trace: [1488089.831306] [] queued_spin_lock_slowpath+0xb/0xf [1488089.831309] [] _raw_spin_lock+0x20/0x30 [1488089.831316] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.831327] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.831365] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.831367] [] ? del_timer_sync+0x52/0x60 [1488089.831404] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.831441] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.831479] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.831481] [] ? wake_up_state+0x20/0x20 [1488089.831519] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.831521] [] kthread+0xd1/0xe0 [1488089.831523] [] ? insert_kthread_work+0x40/0x40 [1488089.831525] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.831527] [] ? insert_kthread_work+0x40/0x40 [1488089.831546] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.850251] NMI watchdog: BUG: soft lockup - CPU#78 stuck for 22s! [ptlrpcd_03_10:54202] [1488089.850277] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.850284] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.850287] CPU: 78 PID: 54202 Comm: ptlrpcd_03_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.850288] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.850289] task: ffff8aaba9650000 ti: ffff8abafca54000 task.ti: ffff8abafca54000 [1488089.850292] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.850293] RSP: 0018:ffff8abafca57b78 EFLAGS: 00000246 [1488089.850294] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002710000 [1488089.850295] RDX: ffff8abaffadb780 RSI: 0000000000f10001 RDI: ffff8adafa7b1b00 [1488089.850296] RBP: ffff8abafca57b78 R08: ffff8abaffc5b780 R09: 0000000000000000 [1488089.850297] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab1562e5400 [1488089.850298] R13: 0005c4aa17ccdf60 R14: ffff8ab2fa371800 R15: 0000000000000007 [1488089.850300] FS: 00007ea1be070700(0000) GS:ffff8abaffc40000(0000) knlGS:0000000000000000 [1488089.850301] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.850302] CR2: 00007ec111ede00b CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.850303] Call Trace: [1488089.850305] [] queued_spin_lock_slowpath+0xb/0xf [1488089.850307] [] _raw_spin_lock+0x20/0x30 [1488089.850315] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.850325] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.850363] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.850365] [] ? del_timer_sync+0x52/0x60 [1488089.850403] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.850440] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.850478] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.850481] [] ? wake_up_state+0x20/0x20 [1488089.850519] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.850521] [] kthread+0xd1/0xe0 [1488089.850522] [] ? insert_kthread_work+0x40/0x40 [1488089.850524] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.850526] [] ? insert_kthread_work+0x40/0x40 [1488089.850546] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.859255] NMI watchdog: BUG: soft lockup - CPU#79 stuck for 22s! [ptlrpcd_07_00:54240] [1488089.859282] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.859289] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.859292] CPU: 79 PID: 54240 Comm: ptlrpcd_07_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.859293] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.859294] task: ffff8abafbc7b0c0 ti: ffff8abaf81e4000 task.ti: ffff8abaf81e4000 [1488089.859297] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.859297] RSP: 0018:ffff8abaf81e7b78 EFLAGS: 00000246 [1488089.859298] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002790000 [1488089.859299] RDX: ffff8a8aff29b780 RSI: 0000000000810001 RDI: ffff8adafa7b1b00 [1488089.859301] RBP: ffff8abaf81e7b78 R08: ffff8afb3fa5b780 R09: 0000000000000000 [1488089.859301] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aedf17a3200 [1488089.859302] R13: 0005c4aa17c00f40 R14: ffff8af7e55f5700 R15: 0000000000000007 [1488089.859304] FS: 00007f480aa7e880(0000) GS:ffff8afb3fa40000(0000) knlGS:0000000000000000 [1488089.859305] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.859306] CR2: 00007ecc283d9000 CR3: 0000006fdd120000 CR4: 00000000003407e0 [1488089.859307] Call Trace: [1488089.859309] [] queued_spin_lock_slowpath+0xb/0xf [1488089.859311] [] _raw_spin_lock+0x20/0x30 [1488089.859319] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.859330] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.859367] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.859405] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.859442] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.859481] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.859483] [] ? wake_up_state+0x20/0x20 [1488089.859521] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.859523] [] kthread+0xd1/0xe0 [1488089.859525] [] ? insert_kthread_work+0x40/0x40 [1488089.859527] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.859529] [] ? insert_kthread_work+0x40/0x40 [1488089.859549] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.867253] NMI watchdog: BUG: soft lockup - CPU#80 stuck for 22s! [ptlrpcd_00_01:54157] [1488089.867280] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.867287] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.867290] CPU: 80 PID: 54157 Comm: ptlrpcd_00_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.867291] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.867292] task: ffff8afaf5f32080 ti: ffff8abaed21c000 task.ti: ffff8abaed21c000 [1488089.867295] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.867296] RSP: 0018:ffff8abaed21fb78 EFLAGS: 00000246 [1488089.867297] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002810000 [1488089.867298] RDX: ffff8abaffcdb780 RSI: 0000000002f10001 RDI: ffff8adafa7b1b00 [1488089.867299] RBP: ffff8abaed21fb78 R08: ffff8a8aff49b780 R09: 0000000000000000 [1488089.867300] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c6a00 [1488089.867301] R13: 0005c4aa187a05c0 R14: ffff8a90d9b10300 R15: 0000000000000007 [1488089.867303] FS: 00007ea1b505e700(0000) GS:ffff8a8aff480000(0000) knlGS:0000000000000000 [1488089.867304] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.867305] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.867306] Call Trace: [1488089.867308] [] queued_spin_lock_slowpath+0xb/0xf [1488089.867310] [] _raw_spin_lock+0x20/0x30 [1488089.867318] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.867328] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.867366] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.867403] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.867440] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.867478] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.867481] [] ? wake_up_state+0x20/0x20 [1488089.867519] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.867521] [] kthread+0xd1/0xe0 [1488089.867522] [] ? insert_kthread_work+0x40/0x40 [1488089.867524] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.867526] [] ? insert_kthread_work+0x40/0x40 [1488089.867546] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.876255] NMI watchdog: BUG: soft lockup - CPU#81 stuck for 22s! [ptlrpcd_04_00:54204] [1488089.876283] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.876290] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.876293] CPU: 81 PID: 54204 Comm: ptlrpcd_04_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.876294] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.876295] task: ffff8aaba9652080 ti: ffff8abaf1608000 task.ti: ffff8abaf1608000 [1488089.876298] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.876299] RSP: 0018:ffff8abaf160bb78 EFLAGS: 00000246 [1488089.876300] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002890000 [1488089.876301] RDX: ffff8afb3f99b780 RSI: 0000000001b90001 RDI: ffff8adafa7b1b00 [1488089.876302] RBP: ffff8abaf160bb78 R08: ffff8acaffc9b780 R09: 0000000000000000 [1488089.876303] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac88f138200 [1488089.876304] R13: 0005c4aa17b5f960 R14: ffff8ac182749500 R15: 0000000000000007 [1488089.876305] FS: 00007ea1dd8af700(0000) GS:ffff8acaffc80000(0000) knlGS:0000000000000000 [1488089.876306] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.876308] CR2: 00007ef77c795f00 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.876308] Call Trace: [1488089.876310] [] queued_spin_lock_slowpath+0xb/0xf [1488089.876312] [] _raw_spin_lock+0x20/0x30 [1488089.876320] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.876331] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.876369] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.876407] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.876444] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.876483] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.876485] [] ? wake_up_state+0x20/0x20 [1488089.876523] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.876525] [] kthread+0xd1/0xe0 [1488089.876527] [] ? insert_kthread_work+0x40/0x40 [1488089.876529] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.876531] [] ? insert_kthread_work+0x40/0x40 [1488089.876551] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.886255] NMI watchdog: BUG: soft lockup - CPU#82 stuck for 22s! [ptlrpcd_01_03:54171] [1488089.886283] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.886291] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.886294] CPU: 82 PID: 54171 Comm: ptlrpcd_01_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.886295] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.886296] task: ffff8abaf6344100 ti: ffff8abafcd88000 task.ti: ffff8abafcd88000 [1488089.886299] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.886300] RSP: 0018:ffff8abafcd8bb78 EFLAGS: 00000246 [1488089.886301] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002910000 [1488089.886302] RDX: ffff8adaffa5b780 RSI: 0000000000590001 RDI: ffff8adafa7b1b00 [1488089.886303] RBP: ffff8abafcd8bb78 R08: ffff8a9affc9b780 R09: 0000000000000000 [1488089.886304] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8cf23c7c00 [1488089.886305] R13: 0005c4aa17a41660 R14: ffff8a97cdd14500 R15: 0000000000000007 [1488089.886307] FS: 00007ea1c9887700(0000) GS:ffff8a9affc80000(0000) knlGS:0000000000000000 [1488089.886308] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.886309] CR2: 00007f8c2c495000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.886310] Call Trace: [1488089.886312] [] queued_spin_lock_slowpath+0xb/0xf [1488089.886313] [] _raw_spin_lock+0x20/0x30 [1488089.886321] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.886332] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.886370] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.886408] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.886445] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.886483] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.886486] [] ? wake_up_state+0x20/0x20 [1488089.886524] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.886526] [] kthread+0xd1/0xe0 [1488089.886528] [] ? insert_kthread_work+0x40/0x40 [1488089.886530] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.886532] [] ? insert_kthread_work+0x40/0x40 [1488089.886552] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.913255] NMI watchdog: BUG: soft lockup - CPU#85 stuck for 22s! [ptlrpcd_06_07:54235] [1488089.913282] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.913290] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.913293] CPU: 85 PID: 54235 Comm: ptlrpcd_06_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.913293] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.913295] task: ffff8abaf4785140 ti: ffff8abafc788000 task.ti: ffff8abafc788000 [1488089.913298] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.913299] RSP: 0018:ffff8abafc78bb78 EFLAGS: 00000246 [1488089.913299] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002a90000 [1488089.913301] RDX: ffff8aeaffa9b780 RSI: 0000000000a90001 RDI: ffff8adafa7b1b00 [1488089.913301] RBP: ffff8abafc78bb78 R08: ffff8aeaffc9b780 R09: 0000000000000000 [1488089.913302] R10: 0000000000000000 R11: 000000000000000f R12: ffff8add4abcd800 [1488089.913304] R13: 0005c4aa17cfee70 R14: ffff8ae7b12db600 R15: 0000000000000007 [1488089.913305] FS: 00007ea1dc8ad700(0000) GS:ffff8aeaffc80000(0000) knlGS:0000000000000000 [1488089.913306] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.913307] CR2: 00007ec44bcee000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.913308] Call Trace: [1488089.913310] [] queued_spin_lock_slowpath+0xb/0xf [1488089.913312] [] _raw_spin_lock+0x20/0x30 [1488089.913320] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.913331] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.913368] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.913406] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.913443] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.913482] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.913485] [] ? wake_up_state+0x20/0x20 [1488089.913523] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.913525] [] kthread+0xd1/0xe0 [1488089.913527] [] ? insert_kthread_work+0x40/0x40 [1488089.913529] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.913531] [] ? insert_kthread_work+0x40/0x40 [1488089.913551] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.923258] NMI watchdog: BUG: soft lockup - CPU#86 stuck for 22s! [ptlrpcd_03_05:54197] [1488089.923286] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.923293] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.923296] CPU: 86 PID: 54197 Comm: ptlrpcd_03_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.923296] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.923298] task: ffff8abaf6392080 ti: ffff8abafab64000 task.ti: ffff8abafab64000 [1488089.923301] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.923302] RSP: 0018:ffff8abafab67b78 EFLAGS: 00000246 [1488089.923303] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002b10000 [1488089.923303] RDX: ffff8adaffcdb780 RSI: 0000000002d90001 RDI: ffff8adafa7b1b00 [1488089.923304] RBP: ffff8abafab67b78 R08: ffff8abaffc9b780 R09: 0000000000000000 [1488089.923306] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aabdc30be00 [1488089.923307] R13: 0005c4aa17d36450 R14: ffff8ab441784b00 R15: 0000000000000007 [1488089.923308] FS: 00007ea1b385b700(0000) GS:ffff8abaffc80000(0000) knlGS:0000000000000000 [1488089.923309] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.923311] CR2: 00007edf3fa26dc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.923311] Call Trace: [1488089.923313] [] queued_spin_lock_slowpath+0xb/0xf [1488089.923315] [] _raw_spin_lock+0x20/0x30 [1488089.923323] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.923334] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.923371] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.923409] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.923445] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.923484] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.923486] [] ? wake_up_state+0x20/0x20 [1488089.923524] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.923526] [] kthread+0xd1/0xe0 [1488089.923528] [] ? insert_kthread_work+0x40/0x40 [1488089.923530] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.923532] [] ? insert_kthread_work+0x40/0x40 [1488089.923552] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488089.932258] NMI watchdog: BUG: soft lockup - CPU#87 stuck for 22s! [ptlrpcd_07_01:54241] [1488089.932285] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.932292] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.932295] CPU: 87 PID: 54241 Comm: ptlrpcd_07_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.932296] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.932297] task: ffff8abafbc7c100 ti: ffff8abaed65c000 task.ti: ffff8abaed65c000 [1488089.932300] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.932301] RSP: 0018:ffff8abaed65fb78 EFLAGS: 00000246 [1488089.932302] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002b90000 [1488089.932303] RDX: ffff8afb3f91b780 RSI: 0000000001390001 RDI: ffff8adafa7b1b00 [1488089.932304] RBP: ffff8abaed65fb78 R08: ffff8afb3fa9b780 R09: 0000000000000000 [1488089.932305] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af635444000 [1488089.932306] R13: 0005c4aa17cd8bb0 R14: ffff8af616504800 R15: 0000000000000007 [1488089.932308] FS: 00007ea1d2098700(0000) GS:ffff8afb3fa80000(0000) knlGS:0000000000000000 [1488089.932309] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.932310] CR2: 00007eec1a4e7dc8 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.932310] Call Trace: [1488089.932312] [] queued_spin_lock_slowpath+0xb/0xf [1488089.932315] [] _raw_spin_lock+0x20/0x30 [1488089.932323] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.932333] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.932371] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.932410] [] ? lustre_msg_buf_v2+0x6e/0x1b0 [ptlrpc] [1488089.932447] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.932484] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.932523] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.932525] [] ? wake_up_state+0x20/0x20 [1488089.932564] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.932566] [] kthread+0xd1/0xe0 [1488089.932567] [] ? insert_kthread_work+0x40/0x40 [1488089.932569] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.932571] [] ? insert_kthread_work+0x40/0x40 [1488089.932591] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.940255] NMI watchdog: BUG: soft lockup - CPU#88 stuck for 22s! [ptlrpcd_00_00:54156] [1488089.940282] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.940290] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.940293] CPU: 88 PID: 54156 Comm: ptlrpcd_00_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.940294] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.940295] task: ffff8afaf5f30000 ti: ffff8abaf9e98000 task.ti: ffff8abaf9e98000 [1488089.940298] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.940299] RSP: 0018:ffff8abaf9e9bb78 EFLAGS: 00000246 [1488089.940300] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002c10000 [1488089.940301] RDX: ffff8aaaffadb780 RSI: 0000000000e10001 RDI: ffff8adafa7b1b00 [1488089.940302] RBP: ffff8abaf9e9bb78 R08: ffff8a8aff4db780 R09: 0000000000000000 [1488089.940303] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa198af4a00 [1488089.940304] R13: 0005c4aa187a16a0 R14: ffff8a97c5ae2a00 R15: 0000000000000007 [1488089.940306] FS: 00007fac50325880(0000) GS:ffff8a8aff4c0000(0000) knlGS:0000000000000000 [1488089.940307] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.940308] CR2: 00007fbad3d81000 CR3: 0000007a97470000 CR4: 00000000003407e0 [1488089.940309] Call Trace: [1488089.940311] [] queued_spin_lock_slowpath+0xb/0xf [1488089.940313] [] _raw_spin_lock+0x20/0x30 [1488089.940321] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.940331] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.940369] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.940406] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.940443] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.940481] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.940483] [] ? wake_up_state+0x20/0x20 [1488089.940521] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.940523] [] kthread+0xd1/0xe0 [1488089.940525] [] ? insert_kthread_work+0x40/0x40 [1488089.940527] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.940529] [] ? insert_kthread_work+0x40/0x40 [1488089.940549] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.949261] NMI watchdog: BUG: soft lockup - CPU#89 stuck for 22s! [ptlrpcd_04_03:54207] [1488089.949289] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.949296] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.949299] CPU: 89 PID: 54207 Comm: ptlrpcd_04_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.949300] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.949301] task: ffff8aaba9655140 ti: ffff8abaed254000 task.ti: ffff8abaed254000 [1488089.949304] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.949305] RSP: 0018:ffff8abaed257b78 EFLAGS: 00000246 [1488089.949306] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002c90000 [1488089.949307] RDX: ffff8adaffb9b780 RSI: 0000000001990001 RDI: ffff8adafa7b1b00 [1488089.949308] RBP: ffff8abaed257b78 R08: ffff8acaffcdb780 R09: 0000000000000000 [1488089.949309] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac1a1ad2400 [1488089.949310] R13: 0005c4aa17cc9980 R14: ffff8ac1a1b0bf00 R15: 0000000000000007 [1488089.949312] FS: 00007ea1df8b3700(0000) GS:ffff8acaffcc0000(0000) knlGS:0000000000000000 [1488089.949313] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.949314] CR2: 00007ee0c37e002a CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.949315] Call Trace: [1488089.949317] [] queued_spin_lock_slowpath+0xb/0xf [1488089.949319] [] _raw_spin_lock+0x20/0x30 [1488089.949327] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.949337] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.949375] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.949414] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.949451] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.949489] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.949491] [] ? wake_up_state+0x20/0x20 [1488089.949530] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.949531] [] kthread+0xd1/0xe0 [1488089.949533] [] ? insert_kthread_work+0x40/0x40 [1488089.949535] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.949537] [] ? insert_kthread_work+0x40/0x40 [1488089.949558] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.976260] NMI watchdog: BUG: soft lockup - CPU#92 stuck for 22s! [ptlrpcd_02_01:54181] [1488089.976287] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.976294] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.976297] CPU: 92 PID: 54181 Comm: ptlrpcd_02_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.976298] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.976299] task: ffff8abaee640000 ti: ffff8abaf8110000 task.ti: ffff8abaf8110000 [1488089.976302] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488089.976303] RSP: 0018:ffff8abaf8113b78 EFLAGS: 00000246 [1488089.976304] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002e10000 [1488089.976305] RDX: ffff8acaffadb780 RSI: 0000000000c90001 RDI: ffff8adafa7b1b00 [1488089.976306] RBP: ffff8abaf8113b78 R08: ffff8aaaffcdb780 R09: 0000000000000000 [1488089.976307] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa9efa17c00 [1488089.976308] R13: 0005c4aa17cf9150 R14: ffff8aa699edf500 R15: 0000000000000007 [1488089.976309] FS: 00007ea1dd0ae700(0000) GS:ffff8aaaffcc0000(0000) knlGS:0000000000000000 [1488089.976310] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.976311] CR2: 00007f8c1f871000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.976312] Call Trace: [1488089.976315] [] queued_spin_lock_slowpath+0xb/0xf [1488089.976317] [] _raw_spin_lock+0x20/0x30 [1488089.976324] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.976335] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.976372] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.976410] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.976446] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.976485] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.976487] [] ? wake_up_state+0x20/0x20 [1488089.976525] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.976527] [] kthread+0xd1/0xe0 [1488089.976529] [] ? insert_kthread_work+0x40/0x40 [1488089.976530] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.976532] [] ? insert_kthread_work+0x40/0x40 [1488089.976552] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488089.985261] NMI watchdog: BUG: soft lockup - CPU#93 stuck for 22s! [ptlrpcd_06_03:54231] [1488089.985289] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488089.985296] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488089.985299] CPU: 93 PID: 54231 Comm: ptlrpcd_06_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488089.985300] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488089.985301] task: ffff8abaf4781040 ti: ffff8abaebfe4000 task.ti: ffff8abaebfe4000 [1488089.985304] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488089.985305] RSP: 0018:ffff8abaebfe7b78 EFLAGS: 00000246 [1488089.985306] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002e90000 [1488089.985307] RDX: ffff8acaffa5b780 RSI: 0000000000490001 RDI: ffff8adafa7b1b00 [1488089.985308] RBP: ffff8abaebfe7b78 R08: ffff8aeaffcdb780 R09: 0000000000000000 [1488089.985309] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adffbec7600 [1488089.985310] R13: 0005c4aa17d15190 R14: ffff8ae87a880f00 R15: 0000000000000007 [1488089.985312] FS: 00007ea1b485d700(0000) GS:ffff8aeaffcc0000(0000) knlGS:0000000000000000 [1488089.985313] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488089.985314] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488089.985315] Call Trace: [1488089.985317] [] queued_spin_lock_slowpath+0xb/0xf [1488089.985319] [] _raw_spin_lock+0x20/0x30 [1488089.985327] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488089.985338] [] LNetMDUnlink+0x78/0x180 [lnet] [1488089.985376] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488089.985413] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488089.985451] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488089.985490] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488089.985492] [] ? wake_up_state+0x20/0x20 [1488089.985530] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488089.985532] [] kthread+0xd1/0xe0 [1488089.985534] [] ? insert_kthread_work+0x40/0x40 [1488089.985536] [] ret_from_fork_nospec_begin+0xe/0x21 [1488089.985538] [] ? insert_kthread_work+0x40/0x40 [1488089.985558] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488090.004264] NMI watchdog: BUG: soft lockup - CPU#95 stuck for 22s! [ptlrpcd_07_04:54244] [1488090.004292] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488090.004299] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488090.004302] CPU: 95 PID: 54244 Comm: ptlrpcd_07_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488090.004302] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488090.004304] task: ffff8abafc520000 ti: ffff8abaf8a88000 task.ti: ffff8abaf8a88000 [1488090.004307] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488090.004308] RSP: 0018:ffff8abaf8a8bb78 EFLAGS: 00000246 [1488090.004309] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002f90000 [1488090.004310] RDX: ffff8aaaffc1b780 RSI: 0000000002210001 RDI: ffff8adafa7b1b00 [1488090.004311] RBP: ffff8abaf8a8bb78 R08: ffff8afb3fadb780 R09: 0000000000000000 [1488090.004312] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aee00395600 [1488090.004313] R13: 0005c4aa17c03340 R14: ffff8afaa4fd9b00 R15: 0000000000000007 [1488090.004314] FS: 00007f7e5fc01700(0000) GS:ffff8afb3fac0000(0000) knlGS:0000000000000000 [1488090.004315] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488090.004317] CR2: 00007ecbd22c6000 CR3: 0000006035e02000 CR4: 00000000003407e0 [1488090.004317] Call Trace: [1488090.004319] [] queued_spin_lock_slowpath+0xb/0xf [1488090.004321] [] _raw_spin_lock+0x20/0x30 [1488090.004329] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488090.004340] [] LNetMDUnlink+0x78/0x180 [lnet] [1488090.004378] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488090.004416] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488090.004453] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488090.004491] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488090.004494] [] ? wake_up_state+0x20/0x20 [1488090.004532] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488090.004533] [] kthread+0xd1/0xe0 [1488090.004535] [] ? insert_kthread_work+0x40/0x40 [1488090.004537] [] ret_from_fork_nospec_begin+0xe/0x21 [1488090.004539] [] ? insert_kthread_work+0x40/0x40 [1488090.004560] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488093.081464] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ptlrpcd_00_08:54164] [1488093.081492] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.081500] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.081503] CPU: 0 PID: 54164 Comm: ptlrpcd_00_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.081504] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.081505] task: ffff8abaf815c100 ti: ffff8abaecad0000 task.ti: ffff8abaecad0000 [1488093.081509] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488093.081510] RSP: 0018:ffff8abaecad3b78 EFLAGS: 00000246 [1488093.081511] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000010000 [1488093.081512] RDX: ffff8a9affc5b780 RSI: 0000000002510001 RDI: ffff8adafa7b1b00 [1488093.081513] RBP: ffff8abaecad3b78 R08: ffff8a8aff21b780 R09: 0000000000000000 [1488093.081514] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9294bafe00 [1488093.081515] R13: 0005c4aa186d9210 R14: ffff8af647b88c00 R15: 0000000000000007 [1488093.081517] FS: 00007ea1dd0ae700(0000) GS:ffff8a8aff200000(0000) knlGS:0000000000000000 [1488093.081518] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.081519] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407f0 [1488093.081520] Call Trace: [1488093.081522] [] queued_spin_lock_slowpath+0xb/0xf [1488093.081524] [] _raw_spin_lock+0x20/0x30 [1488093.081532] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.081543] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.081580] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.081618] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.081655] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.081693] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.081695] [] ? wake_up_state+0x20/0x20 [1488093.081733] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.081735] [] kthread+0xd1/0xe0 [1488093.081737] [] ? insert_kthread_work+0x40/0x40 [1488093.081739] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.081741] [] ? insert_kthread_work+0x40/0x40 [1488093.081761] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488093.311480] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ptlrpcd_05_02:54218] [1488093.311507] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.311514] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.311516] CPU: 19 PID: 54218 Comm: ptlrpcd_05_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.311517] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.311518] task: ffff8abaee63a080 ti: ffff8abaecb24000 task.ti: ffff8abaecb24000 [1488093.311521] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488093.311522] RSP: 0018:ffff8abaecb27b78 EFLAGS: 00000246 [1488093.311523] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000990000 [1488093.311523] RDX: ffff8aaaffc9b780 RSI: 0000000002a10001 RDI: ffff8adafa7b1b00 [1488093.311524] RBP: ffff8abaecb27b78 R08: ffff8adaffa9b780 R09: 0000000000000000 [1488093.311525] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad0543e2a00 [1488093.311525] R13: 0005c4aa1786d7b0 R14: ffff8adab3c4b900 R15: 0000000000000007 [1488093.311527] FS: 00007ea1bc06c700(0000) GS:ffff8adaffa80000(0000) knlGS:0000000000000000 [1488093.311528] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.311528] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.311529] Call Trace: [1488093.311531] [] queued_spin_lock_slowpath+0xb/0xf [1488093.311533] [] _raw_spin_lock+0x20/0x30 [1488093.311542] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.311552] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.311590] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.311592] [] ? del_timer_sync+0x52/0x60 [1488093.311629] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.311665] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.311703] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.311705] [] ? wake_up_state+0x20/0x20 [1488093.311743] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.311744] [] kthread+0xd1/0xe0 [1488093.311746] [] ? insert_kthread_work+0x40/0x40 [1488093.311748] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.311749] [] ? insert_kthread_work+0x40/0x40 [1488093.311768] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488093.357479] NMI watchdog: BUG: soft lockup - CPU#24 stuck for 22s! [ptlrpcd_00_11:54167] [1488093.357507] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.357515] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.357517] CPU: 24 PID: 54167 Comm: ptlrpcd_00_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.357518] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.357519] task: ffff8abaf6340000 ti: ffff8abafcf38000 task.ti: ffff8abafcf38000 [1488093.357522] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488093.357523] RSP: 0018:ffff8abafcf3bb78 EFLAGS: 00000246 [1488093.357524] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000c10000 [1488093.357525] RDX: ffff8a9affcdb780 RSI: 0000000002d10001 RDI: ffff8adafa7b1b00 [1488093.357526] RBP: ffff8abafcf3bb78 R08: ffff8a8aff2db780 R09: 0000000000000000 [1488093.357527] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c7c00 [1488093.357528] R13: 0005c4aa187a0800 R14: ffff8a9a09e6d400 R15: 0000000000000007 [1488093.357530] FS: 00007ea1b9867700(0000) GS:ffff8a8aff2c0000(0000) knlGS:0000000000000000 [1488093.357531] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.357532] CR2: 00007f03faf61003 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.357533] Call Trace: [1488093.357535] [] queued_spin_lock_slowpath+0xb/0xf [1488093.357537] [] _raw_spin_lock+0x20/0x30 [1488093.357545] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.357556] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.357593] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.357595] [] ? del_timer_sync+0x52/0x60 [1488093.357632] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.357669] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.357708] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.357710] [] ? wake_up_state+0x20/0x20 [1488093.357748] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.357749] [] kthread+0xd1/0xe0 [1488093.357751] [] ? insert_kthread_work+0x40/0x40 [1488093.357753] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.357755] [] ? insert_kthread_work+0x40/0x40 [1488093.357775] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488093.457490] NMI watchdog: BUG: soft lockup - CPU#35 stuck for 23s! [ptlrpcd_05_09:54225] [1488093.457517] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.457524] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.457527] CPU: 35 PID: 54225 Comm: ptlrpcd_05_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.457527] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.457528] task: ffff8abafb33a080 ti: ffff8abafc488000 task.ti: ffff8abafc488000 [1488093.457531] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488093.457532] RSP: 0018:ffff8abafc48bb78 EFLAGS: 00000246 [1488093.457533] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001190000 [1488093.457533] RDX: ffff8a8aff41b780 RSI: 0000000002010001 RDI: ffff8adafa7b1b00 [1488093.457534] RBP: ffff8abafc48bb78 R08: ffff8adaffb1b780 R09: 0000000000000000 [1488093.457535] R10: 0000000000000000 R11: 000000000000000f R12: ffff8accfe737200 [1488093.457535] R13: 0005c4aa17a87530 R14: ffff8ad2bf7d4800 R15: 0000000000000007 [1488093.457537] FS: 00007ea1d489d700(0000) GS:ffff8adaffb00000(0000) knlGS:0000000000000000 [1488093.457537] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.457538] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.457539] Call Trace: [1488093.457541] [] queued_spin_lock_slowpath+0xb/0xf [1488093.457543] [] _raw_spin_lock+0x20/0x30 [1488093.457550] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.457561] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.457598] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.457600] [] ? del_timer_sync+0x52/0x60 [1488093.457637] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.457674] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.457712] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.457714] [] ? wake_up_state+0x20/0x20 [1488093.457753] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.457754] [] kthread+0xd1/0xe0 [1488093.457756] [] ? insert_kthread_work+0x40/0x40 [1488093.457758] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.457760] [] ? insert_kthread_work+0x40/0x40 [1488093.457779] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488093.577498] NMI watchdog: BUG: soft lockup - CPU#48 stuck for 23s! [ptlrpcd_00_05:54161] [1488093.577525] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.577533] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.577535] CPU: 48 PID: 54161 Comm: ptlrpcd_00_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.577536] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.577537] task: ffff8abaf8159040 ti: ffff8abafc6a8000 task.ti: ffff8abafc6a8000 [1488093.577540] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488093.577541] RSP: 0018:ffff8abafc6abb78 EFLAGS: 00000246 [1488093.577542] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001810000 [1488093.577543] RDX: ffff8adaffbdb780 RSI: 0000000001d90001 RDI: ffff8adafa7b1b00 [1488093.577544] RBP: ffff8abafc6abb78 R08: ffff8a8aff39b780 R09: 0000000000000000 [1488093.577545] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c0400 [1488093.577547] R13: 0005c4aa1879fad0 R14: ffff8a7b02ec1800 R15: 0000000000000007 [1488093.577549] FS: 00007ea1af052700(0000) GS:ffff8a8aff380000(0000) knlGS:0000000000000000 [1488093.577550] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.577551] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.577552] Call Trace: [1488093.577554] [] queued_spin_lock_slowpath+0xb/0xf [1488093.577556] [] _raw_spin_lock+0x20/0x30 [1488093.577564] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.577574] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.577612] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.577649] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.577686] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.577724] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.577726] [] ? wake_up_state+0x20/0x20 [1488093.577764] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.577766] [] kthread+0xd1/0xe0 [1488093.577768] [] ? insert_kthread_work+0x40/0x40 [1488093.577771] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.577772] [] ? insert_kthread_work+0x40/0x40 [1488093.577792] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488093.649498] NMI watchdog: BUG: soft lockup - CPU#56 stuck for 23s! [ptlrpcd_00_03:54159] [1488093.649525] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.649533] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.649535] CPU: 56 PID: 54159 Comm: ptlrpcd_00_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.649536] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.649537] task: ffff8afaf5f36180 ti: ffff8abafb264000 task.ti: ffff8abafb264000 [1488093.649540] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488093.649541] RSP: 0018:ffff8abafb267b78 EFLAGS: 00000246 [1488093.649543] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001c10000 [1488093.649543] RDX: ffff8abaffb9b780 RSI: 0000000001b10001 RDI: ffff8adafa7b1b00 [1488093.649544] RBP: ffff8abafb267b78 R08: ffff8a8aff3db780 R09: 0000000000000000 [1488093.649545] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c2c00 [1488093.649546] R13: 0005c4aa1879ff30 R14: ffff8a90d9b12d00 R15: 0000000000000007 [1488093.649548] FS: 00007fc0d2114880(0000) GS:ffff8a8aff3c0000(0000) knlGS:0000000000000000 [1488093.649549] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.649550] CR2: 00007f012f4f9005 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.649551] Call Trace: [1488093.649553] [] queued_spin_lock_slowpath+0xb/0xf [1488093.649555] [] _raw_spin_lock+0x20/0x30 [1488093.649563] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.649574] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.649611] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.649649] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.649687] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.649726] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.649728] [] ? wake_up_state+0x20/0x20 [1488093.649767] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.649769] [] kthread+0xd1/0xe0 [1488093.649770] [] ? insert_kthread_work+0x40/0x40 [1488093.649772] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.649774] [] ? insert_kthread_work+0x40/0x40 [1488093.649794] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488093.694504] NMI watchdog: BUG: soft lockup - CPU#61 stuck for 23s! [ptlrpcd_06_08:54236] [1488093.694532] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.694540] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.694542] CPU: 61 PID: 54236 Comm: ptlrpcd_06_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.694543] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.694544] task: ffff8abaf4786180 ti: ffff8abaeb6a8000 task.ti: ffff8abaeb6a8000 [1488093.694548] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488093.694548] RSP: 0018:ffff8abaeb6abb78 EFLAGS: 00000246 [1488093.694549] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001e90000 [1488093.694550] RDX: ffff8a9affbdb780 RSI: 0000000001d10001 RDI: ffff8adafa7b1b00 [1488093.694552] RBP: ffff8abaeb6abb78 R08: ffff8aeaffbdb780 R09: 0000000000000000 [1488093.694553] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adeb78e0200 [1488093.694554] R13: 0005c4aa17c27490 R14: ffff8ab98a57ec00 R15: 0000000000000007 [1488093.694556] FS: 00007ea1b585f700(0000) GS:ffff8aeaffbc0000(0000) knlGS:0000000000000000 [1488093.694556] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.694557] CR2: 00007edf22602002 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.694558] Call Trace: [1488093.694560] [] queued_spin_lock_slowpath+0xb/0xf [1488093.694562] [] _raw_spin_lock+0x20/0x30 [1488093.694570] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.694581] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.694619] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.694621] [] ? del_timer_sync+0x52/0x60 [1488093.694659] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.694696] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.694734] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.694737] [] ? wake_up_state+0x20/0x20 [1488093.694775] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.694777] [] kthread+0xd1/0xe0 [1488093.694779] [] ? insert_kthread_work+0x40/0x40 [1488093.694781] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.694783] [] ? insert_kthread_work+0x40/0x40 [1488093.694803] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488093.750506] NMI watchdog: BUG: soft lockup - CPU#67 stuck for 23s! [ptlrpcd_05_07:54223] [1488093.750533] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488093.750540] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488093.750542] CPU: 67 PID: 54223 Comm: ptlrpcd_05_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488093.750542] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488093.750543] task: ffff8abafb338000 ti: ffff8abafc59c000 task.ti: ffff8abafc59c000 [1488093.750546] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488093.750547] RSP: 0018:ffff8abafc59fb78 EFLAGS: 00000246 [1488093.750547] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002190000 [1488093.750548] RDX: ffff8a8aff31b780 RSI: 0000000001010001 RDI: ffff8adafa7b1b00 [1488093.750549] RBP: ffff8abafc59fb78 R08: ffff8adaffc1b780 R09: 0000000000000000 [1488093.750549] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad080454600 [1488093.750550] R13: 0005c4aa17af5920 R14: ffff8ad4ecdef800 R15: 0000000000000007 [1488093.750551] FS: 00007ea1da8a9700(0000) GS:ffff8adaffc00000(0000) knlGS:0000000000000000 [1488093.750552] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488093.750553] CR2: 00007f3af9191ac0 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488093.750554] Call Trace: [1488093.750555] [] queued_spin_lock_slowpath+0xb/0xf [1488093.750557] [] _raw_spin_lock+0x20/0x30 [1488093.750565] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488093.750575] [] LNetMDUnlink+0x78/0x180 [lnet] [1488093.750613] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488093.750650] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488093.750687] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488093.750725] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488093.750727] [] ? wake_up_state+0x20/0x20 [1488093.750765] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488093.750767] [] kthread+0xd1/0xe0 [1488093.750768] [] ? insert_kthread_work+0x40/0x40 [1488093.750770] [] ret_from_fork_nospec_begin+0xe/0x21 [1488093.750771] [] ? insert_kthread_work+0x40/0x40 [1488093.750791] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488097.814776] NMI watchdog: BUG: soft lockup - CPU#74 stuck for 23s! [ptlrpcd_01_08:54176] [1488097.814802] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488097.814810] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488097.814812] CPU: 74 PID: 54176 Comm: ptlrpcd_01_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488097.814813] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488097.814814] task: ffff8abaf4daa080 ti: ffff8abaf63f0000 task.ti: ffff8abaf63f0000 [1488097.814817] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488097.814819] RSP: 0018:ffff8abaf63f3b78 EFLAGS: 00000246 [1488097.814820] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002510000 [1488097.814821] RDX: ffff8aaaffb5b780 RSI: 0000000001610001 RDI: ffff8adafa7b1b00 [1488097.814821] RBP: ffff8abaf63f3b78 R08: ffff8a9affc5b780 R09: 0000000000000000 [1488097.814823] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a95b8e0f400 [1488097.814824] R13: 0005c4aa179d8f00 R14: ffff8a8fdf5eb300 R15: 0000000000000007 [1488097.814825] FS: 00007ea1d409c700(0000) GS:ffff8a9affc40000(0000) knlGS:0000000000000000 [1488097.814826] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488097.814828] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488097.814828] Call Trace: [1488097.814830] [] queued_spin_lock_slowpath+0xb/0xf [1488097.814832] [] _raw_spin_lock+0x20/0x30 [1488097.814840] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488097.814851] [] LNetMDUnlink+0x78/0x180 [lnet] [1488097.814889] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488097.814891] [] ? del_timer_sync+0x52/0x60 [1488097.814929] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488097.814966] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488097.815004] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488097.815007] [] ? wake_up_state+0x20/0x20 [1488097.815045] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488097.815047] [] kthread+0xd1/0xe0 [1488097.815049] [] ? insert_kthread_work+0x40/0x40 [1488097.815051] [] ret_from_fork_nospec_begin+0xe/0x21 [1488097.815052] [] ? insert_kthread_work+0x40/0x40 [1488097.815073] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488097.895777] NMI watchdog: BUG: soft lockup - CPU#83 stuck for 23s! [ptlrpcd_05_05:54221] [1488097.895805] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488097.895812] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488097.895814] CPU: 83 PID: 54221 Comm: ptlrpcd_05_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488097.895815] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488097.895816] task: ffff8abaee63d140 ti: ffff8abaf87c8000 task.ti: ffff8abaf87c8000 [1488097.895819] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488097.895820] RSP: 0018:ffff8abaf87cbb78 EFLAGS: 00000246 [1488097.895820] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002990000 [1488097.895821] RDX: ffff8abaffa5b780 RSI: 0000000000710001 RDI: ffff8adafa7b1b00 [1488097.895822] RBP: ffff8abaf87cbb78 R08: ffff8adaffc9b780 R09: 0000000000000000 [1488097.895822] R10: 0000000000000000 R11: 000000000000000f R12: ffff8acba1d9a000 [1488097.895823] R13: 0005c4aa17ad97e0 R14: ffff8ad6e02a9500 R15: 0000000000000007 [1488097.895824] FS: 00007ea1cc08c700(0000) GS:ffff8adaffc80000(0000) knlGS:0000000000000000 [1488097.895825] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488097.895826] CR2: 00007f8c121f1000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488097.895826] Call Trace: [1488097.895828] [] queued_spin_lock_slowpath+0xb/0xf [1488097.895830] [] _raw_spin_lock+0x20/0x30 [1488097.895838] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488097.895848] [] LNetMDUnlink+0x78/0x180 [lnet] [1488097.895886] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488097.895923] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488097.895960] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488097.895999] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488097.896000] [] ? wake_up_state+0x20/0x20 [1488097.896039] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488097.896040] [] kthread+0xd1/0xe0 [1488097.896042] [] ? insert_kthread_work+0x40/0x40 [1488097.896044] [] ret_from_fork_nospec_begin+0xe/0x21 [1488097.896045] [] ? insert_kthread_work+0x40/0x40 [1488097.896065] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488097.959783] NMI watchdog: BUG: soft lockup - CPU#90 stuck for 23s! [ptlrpcd_01_06:54174] [1488097.959811] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488097.959818] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488097.959821] CPU: 90 PID: 54174 Comm: ptlrpcd_01_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488097.959822] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488097.959823] task: ffff8abaf4da8000 ti: ffff8abaf8f0c000 task.ti: ffff8abaf8f0c000 [1488097.959826] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488097.959827] RSP: 0018:ffff8abaf8f0fb78 EFLAGS: 00000246 [1488097.959828] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002d10000 [1488097.959829] RDX: ffff8a8aff35b780 RSI: 0000000001410001 RDI: ffff8adafa7b1b00 [1488097.959830] RBP: ffff8abaf8f0fb78 R08: ffff8a9affcdb780 R09: 0000000000000000 [1488097.959831] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9800c84200 [1488097.959832] R13: 0005c4aa179e6300 R14: ffff8a8ffc87d100 R15: 0000000000000007 [1488097.959834] FS: 00007ea1b9867700(0000) GS:ffff8a9affcc0000(0000) knlGS:0000000000000000 [1488097.959835] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488097.959836] CR2: 00007fb3d30fe000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488097.959837] Call Trace: [1488097.959839] [] queued_spin_lock_slowpath+0xb/0xf [1488097.959841] [] _raw_spin_lock+0x20/0x30 [1488097.959849] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488097.959860] [] LNetMDUnlink+0x78/0x180 [lnet] [1488097.959897] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488097.959935] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488097.959971] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488097.960010] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488097.960012] [] ? wake_up_state+0x20/0x20 [1488097.960050] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488097.960052] [] kthread+0xd1/0xe0 [1488097.960054] [] ? insert_kthread_work+0x40/0x40 [1488097.960056] [] ret_from_fork_nospec_begin+0xe/0x21 [1488097.960058] [] ? insert_kthread_work+0x40/0x40 [1488097.960078] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488097.968787] NMI watchdog: BUG: soft lockup - CPU#91 stuck for 23s! [ptlrpcd_05_04:54220] [1488097.968814] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488097.968822] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488097.968824] CPU: 91 PID: 54220 Comm: ptlrpcd_05_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488097.968825] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488097.968826] task: ffff8abaee63c100 ti: ffff8abaff6ec000 task.ti: ffff8abaff6ec000 [1488097.968830] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488097.968831] RSP: 0018:ffff8abaff6efb78 EFLAGS: 00000246 [1488097.968832] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002d90000 [1488097.968833] RDX: ffff8abaffb5b780 RSI: 0000000001710001 RDI: ffff8adafa7b1b00 [1488097.968834] RBP: ffff8abaff6efb78 R08: ffff8adaffcdb780 R09: 0000000000000000 [1488097.968835] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad86dcdb000 [1488097.968836] R13: 0005c4aa17a95ea0 R14: ffff8accdd36aa00 R15: 0000000000000007 [1488097.968838] FS: 00007ea1b9066700(0000) GS:ffff8adaffcc0000(0000) knlGS:0000000000000000 [1488097.968838] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488097.968840] CR2: 00007f8c1f871000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488097.968840] Call Trace: [1488097.968843] [] queued_spin_lock_slowpath+0xb/0xf [1488097.968845] [] _raw_spin_lock+0x20/0x30 [1488097.968853] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488097.968864] [] LNetMDUnlink+0x78/0x180 [lnet] [1488097.968902] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488097.968904] [] ? del_timer_sync+0x52/0x60 [1488097.968941] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488097.968979] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488097.969017] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488097.969019] [] ? wake_up_state+0x20/0x20 [1488097.969058] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488097.969060] [] kthread+0xd1/0xe0 [1488097.969061] [] ? insert_kthread_work+0x40/0x40 [1488097.969063] [] ret_from_fork_nospec_begin+0xe/0x21 [1488097.969065] [] ? insert_kthread_work+0x40/0x40 [1488097.969085] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488101.149991] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ptlrpcd_04_10:54214] [1488101.150020] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488101.150027] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488101.150030] CPU: 1 PID: 54214 Comm: ptlrpcd_04_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488101.150031] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488101.150032] task: ffff8abaf82cd140 ti: ffff8abaf4ad4000 task.ti: ffff8abaf4ad4000 [1488101.150035] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488101.150036] RSP: 0018:ffff8abaf4ad7b78 EFLAGS: 00000246 [1488101.150038] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000090000 [1488101.150038] RDX: ffff8acaffb9b780 RSI: 0000000001890001 RDI: ffff8adafa7b1b00 [1488101.150040] RBP: ffff8abaf4ad7b78 R08: ffff8acaffa1b780 R09: 0000000000000000 [1488101.150041] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ac248730000 [1488101.150041] R13: 0005c4aa17c86390 R14: ffff8ac9ade81200 R15: 0000000000000007 [1488101.150043] FS: 00007f8eefb88700(0000) GS:ffff8acaffa00000(0000) knlGS:0000000000000000 [1488101.150044] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488101.150045] CR2: 00007f8eefd2e000 CR3: 0000004039b7e000 CR4: 00000000003407e0 [1488101.150046] Call Trace: [1488101.150048] [] queued_spin_lock_slowpath+0xb/0xf [1488101.150050] [] _raw_spin_lock+0x20/0x30 [1488101.150058] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488101.150069] [] LNetMDUnlink+0x78/0x180 [lnet] [1488101.150107] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488101.150109] [] ? del_timer_sync+0x52/0x60 [1488101.150147] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488101.150184] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488101.150223] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488101.150225] [] ? wake_up_state+0x20/0x20 [1488101.150264] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488101.150265] [] kthread+0xd1/0xe0 [1488101.150267] [] ? insert_kthread_work+0x40/0x40 [1488101.150269] [] ret_from_fork_nospec_begin+0xe/0x21 [1488101.150271] [] ? insert_kthread_work+0x40/0x40 [1488101.150291] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488101.395008] NMI watchdog: BUG: soft lockup - CPU#28 stuck for 22s! [ptlrpcd_02_05:54185] [1488101.395035] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488101.395043] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488101.395046] CPU: 28 PID: 54185 Comm: ptlrpcd_02_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488101.395047] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488101.395048] task: ffff8abaee644100 ti: ffff8abafa6f4000 task.ti: ffff8abafa6f4000 [1488101.395051] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488101.395052] RSP: 0018:ffff8abafa6f7b78 EFLAGS: 00000246 [1488101.395053] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000e10000 [1488101.395054] RDX: ffff8a8aff25b780 RSI: 0000000000410001 RDI: ffff8adafa7b1b00 [1488101.395055] RBP: ffff8abafa6f7b78 R08: ffff8aaaffadb780 R09: 0000000000000000 [1488101.395056] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa144f21200 [1488101.395057] R13: 0005c4aa180e0f10 R14: ffff8aa4fd689800 R15: 0000000000000007 [1488101.395059] FS: 00007ea1ba068700(0000) GS:ffff8aaaffac0000(0000) knlGS:0000000000000000 [1488101.395060] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488101.395061] CR2: 00007f8c22c61000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488101.395062] Call Trace: [1488101.395064] [] queued_spin_lock_slowpath+0xb/0xf [1488101.395066] [] _raw_spin_lock+0x20/0x30 [1488101.395074] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488101.395085] [] LNetMDUnlink+0x78/0x180 [lnet] [1488101.395123] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488101.395161] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488101.395198] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488101.395237] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488101.395239] [] ? wake_up_state+0x20/0x20 [1488101.395277] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488101.395279] [] kthread+0xd1/0xe0 [1488101.395281] [] ? insert_kthread_work+0x40/0x40 [1488101.395283] [] ret_from_fork_nospec_begin+0xe/0x21 [1488101.395285] [] ? insert_kthread_work+0x40/0x40 [1488101.395306] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488101.513014] NMI watchdog: BUG: soft lockup - CPU#41 stuck for 22s! [ptlrpcd_04_11:54215] [1488101.513042] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488101.513049] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488101.513052] CPU: 41 PID: 54215 Comm: ptlrpcd_04_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488101.513052] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488101.513054] task: ffff8abaf82ce180 ti: ffff8abafc750000 task.ti: ffff8abafc750000 [1488101.513057] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488101.513058] RSP: 0018:ffff8abafc753b78 EFLAGS: 00000246 [1488101.513059] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001490000 [1488101.513060] RDX: ffff8aaaffa5b780 RSI: 0000000000610001 RDI: ffff8adafa7b1b00 [1488101.513061] RBP: ffff8abafc753b78 R08: ffff8acaffb5b780 R09: 0000000000000000 [1488101.513062] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abbeeaa5a00 [1488101.513063] R13: 0005c4aa17cd2ea0 R14: ffff8abbeea05700 R15: 0000000000000007 [1488101.513065] FS: 00007ea1b0054700(0000) GS:ffff8acaffb40000(0000) knlGS:0000000000000000 [1488101.513066] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488101.513067] CR2: 00007f8c1b471000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488101.513068] Call Trace: [1488101.513070] [] queued_spin_lock_slowpath+0xb/0xf [1488101.513072] [] _raw_spin_lock+0x20/0x30 [1488101.513080] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488101.513091] [] LNetMDUnlink+0x78/0x180 [lnet] [1488101.513130] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488101.513167] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488101.513204] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488101.513243] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488101.513245] [] ? wake_up_state+0x20/0x20 [1488101.513284] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488101.513286] [] kthread+0xd1/0xe0 [1488101.513288] [] ? insert_kthread_work+0x40/0x40 [1488101.513290] [] ret_from_fork_nospec_begin+0xe/0x21 [1488101.513292] [] ? insert_kthread_work+0x40/0x40 [1488101.513311] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488101.623029] NMI watchdog: BUG: soft lockup - CPU#53 stuck for 22s! [ptlrpcd_06_09:54237] [1488101.623057] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488101.623064] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488101.623067] CPU: 53 PID: 54237 Comm: ptlrpcd_06_09 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488101.623068] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488101.623069] task: ffff8abafbc78000 ti: ffff8abafbbec000 task.ti: ffff8abafbbec000 [1488101.623072] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488101.623073] RSP: 0018:ffff8abafbbefb78 EFLAGS: 00000246 [1488101.623074] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001a90000 [1488101.623075] RDX: ffff8acaffc9b780 RSI: 0000000002890001 RDI: ffff8adafa7b1b00 [1488101.623076] RBP: ffff8abafbbefb78 R08: ffff8aeaffb9b780 R09: 0000000000000000 [1488101.623077] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ae286868200 [1488101.623078] R13: 0005c4aa18160ad0 R14: ffff8ae92e9aaa00 R15: 0000000000000007 [1488101.623080] FS: 00007ea1d60a0700(0000) GS:ffff8aeaffb80000(0000) knlGS:0000000000000000 [1488101.623081] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488101.623082] CR2: 00007eb6dd85a000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488101.623083] Call Trace: [1488101.623085] [] queued_spin_lock_slowpath+0xb/0xf [1488101.623088] [] _raw_spin_lock+0x20/0x30 [1488101.623096] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488101.623106] [] LNetMDUnlink+0x78/0x180 [lnet] [1488101.623144] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488101.623182] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488101.623219] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488101.623258] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488101.623260] [] ? wake_up_state+0x20/0x20 [1488101.623299] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488101.623300] [] kthread+0xd1/0xe0 [1488101.623302] [] ? insert_kthread_work+0x40/0x40 [1488101.623304] [] ret_from_fork_nospec_begin+0xe/0x21 [1488101.623306] [] ? insert_kthread_work+0x40/0x40 [1488101.623326] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488101.841036] NMI watchdog: BUG: soft lockup - CPU#77 stuck for 22s! [ptlrpcd_06_04:54232] [1488101.841064] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488101.841071] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488101.841074] CPU: 77 PID: 54232 Comm: ptlrpcd_06_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488101.841075] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488101.841076] task: ffff8abaf4782080 ti: ffff8abafbecc000 task.ti: ffff8abafbecc000 [1488101.841080] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488101.841081] RSP: 0018:ffff8abafbecfb78 EFLAGS: 00000246 [1488101.841082] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002690000 [1488101.841083] RDX: ffff8a8aff4db780 RSI: 0000000002c10001 RDI: ffff8adafa7b1b00 [1488101.841085] RBP: ffff8abafbecfb78 R08: ffff8aeaffc5b780 R09: 0000000000000000 [1488101.841085] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ae328ff7a00 [1488101.841086] R13: 0005c4aa17cbee60 R14: ffff8add42fe9800 R15: 0000000000000007 [1488101.841088] FS: 00007ea1d2098700(0000) GS:ffff8aeaffc40000(0000) knlGS:0000000000000000 [1488101.841089] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488101.841090] CR2: 00007fc0cc5b7198 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488101.841091] Call Trace: [1488101.841093] [] queued_spin_lock_slowpath+0xb/0xf [1488101.841095] [] _raw_spin_lock+0x20/0x30 [1488101.841103] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488101.841114] [] LNetMDUnlink+0x78/0x180 [lnet] [1488101.841152] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488101.841190] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488101.841227] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488101.841265] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488101.841268] [] ? wake_up_state+0x20/0x20 [1488101.841306] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488101.841308] [] kthread+0xd1/0xe0 [1488101.841310] [] ? insert_kthread_work+0x40/0x40 [1488101.841312] [] ret_from_fork_nospec_begin+0xe/0x21 [1488101.841314] [] ? insert_kthread_work+0x40/0x40 [1488101.841334] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488104.992248] INFO: rcu_sched self-detected stall on CPU [1488104.992251] INFO: rcu_sched self-detected stall on CPU [1488104.992253] INFO: rcu_sched self-detected stall on CPU [1488104.992256] INFO: rcu_sched self-detected stall on CPU [1488104.992260] INFO: rcu_sched self-detected stall on CPU [1488104.992261] INFO: rcu_sched self-detected stall on CPU [1488104.992262] INFO: rcu_sched self-detected stall on CPU [1488104.992264] INFO: rcu_sched self-detected stall on CPU [1488104.992264] INFO: rcu_sched self-detected stall on CPU [1488104.992268] INFO: rcu_sched self-detected stall on CPU [1488104.992269] INFO: rcu_sched self-detected stall on CPU [1488104.992272] INFO: rcu_sched self-detected stall on CPU [1488104.992273] INFO: rcu_sched self-detected stall on CPU [1488104.992274] INFO: rcu_sched self-detected stall on CPU [1488104.992277] INFO: rcu_sched self-detected stall on CPU [1488104.992281] INFO: rcu_sched self-detected stall on CPU [1488104.992282] INFO: rcu_sched self-detected stall on CPU [1488104.992283] { [1488104.992284] { [1488104.992285] { [1488104.992288] { [1488104.992291] { [1488104.992291] { [1488104.992292] { [1488104.992293] { [1488104.992293] { [1488104.992295] { [1488104.992296] { [1488104.992296] { [1488104.992297] { [1488104.992298] { [1488104.992299] { [1488104.992302] { [1488104.992304] { [1488104.992304] 40 [1488104.992306] 56 [1488104.992308] 26 [1488104.992311] 8 [1488104.992314] 43 [1488104.992315] 80 [1488104.992315] 48 [1488104.992316] 0 [1488104.992317] 16 [1488104.992319] 64 [1488104.992320] 13 [1488104.992321] 61 [1488104.992323] 5 [1488104.992324] 21 [1488104.992326] 69 [1488104.992329] 19 [1488104.992330] 72 [1488104.992330] } [1488104.992331] } [1488104.992333] } [1488104.992336] } [1488104.992338] } [1488104.992339] } [1488104.992339] } [1488104.992340] } [1488104.992341] } [1488104.992342] } [1488104.992343] } [1488104.992345] } [1488104.992346] } [1488104.992346] } [1488104.992349] } [1488104.992351] } [1488104.992354] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992356] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992358] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992361] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992364] Task dump for CPU 0: [1488104.992368] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992372] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992374] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992376] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992377] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992379] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992382] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992384] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992386] } [1488104.992391] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992393] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992394] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992397] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992401] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.992403] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488104.992404] Call Trace: [1488104.992443] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.992480] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.992517] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.992555] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.992557] [] ? wake_up_state+0x20/0x20 [1488104.992595] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.992597] [] ? kthread+0xd1/0xe0 [1488104.992599] [] ? insert_kthread_work+0x40/0x40 [1488104.992601] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.992603] [] ? insert_kthread_work+0x40/0x40 [1488104.992604] Task dump for CPU 5: [1488104.992607] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488104.992607] Call Trace: [1488104.992644] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.992682] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.992719] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.992757] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.992759] [] ? wake_up_state+0x20/0x20 [1488104.992797] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.992798] [] ? kthread+0xd1/0xe0 [1488104.992801] [] ? insert_kthread_work+0x40/0x40 [1488104.992803] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.992804] [] ? insert_kthread_work+0x40/0x40 [1488104.992805] Task dump for CPU 8: [1488104.992807] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488104.992807] Call Trace: [1488104.992845] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.992882] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.992918] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.992956] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.992958] [] ? wake_up_state+0x20/0x20 [1488104.992996] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.992998] [] ? kthread+0xd1/0xe0 [1488104.992999] [] ? insert_kthread_work+0x40/0x40 [1488104.993001] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.993003] [] ? insert_kthread_work+0x40/0x40 [1488104.993004] Task dump for CPU 13: [1488104.993007] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488104.993007] Call Trace: [1488104.993045] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.993082] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.993119] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.993157] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.993159] [] ? wake_up_state+0x20/0x20 [1488104.993197] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.993198] [] ? kthread+0xd1/0xe0 [1488104.993200] [] ? insert_kthread_work+0x40/0x40 [1488104.993202] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.993204] [] ? insert_kthread_work+0x40/0x40 [1488104.993205] Task dump for CPU 16: [1488104.993205] Task dump for CPU 0: [1488104.993207] ptlrpcd_00_07 R [1488104.993208] ptlrpcd_00_08 R [1488104.993208] running task [1488104.993210] 0 54163 2 0x00000088 [1488104.993211] running task 0 54164 2 0x00000088 [1488104.993212] Call Trace: [1488104.993213] Call Trace: [1488104.993251] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.993290] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.993295] INFO: rcu_sched self-detected stall on CPU [1488104.993296] INFO: rcu_sched self-detected stall on CPU [1488104.993298] INFO: rcu_sched self-detected stall on CPU [1488104.993302] INFO: rcu_sched self-detected stall on CPU [1488104.993303] INFO: rcu_sched self-detected stall on CPU [1488104.993305] INFO: rcu_sched self-detected stall on CPU [1488104.993344] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.993383] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.993385] INFO: rcu_sched self-detected stall on CPU [1488104.993387] { [1488104.993388] { [1488104.993389] { [1488104.993391] { [1488104.993392] { [1488104.993393] { [1488104.993432] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.993469] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.993471] { [1488104.993473] 42 [1488104.993474] 24 [1488104.993476] 88 [1488104.993478] 32 [1488104.993479] 95 [1488104.993480] 47 [1488104.993520] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.993560] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.993561] 79 [1488104.993562] } [1488104.993563] } [1488104.993565] } [1488104.993566] } [1488104.993567] } [1488104.993568] } [1488104.993573] [] ? wake_up_state+0x20/0x20 [1488104.993576] [] ? wake_up_state+0x20/0x20 [1488104.993614] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.993653] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.993656] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993659] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993661] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993663] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993667] } (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993668] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993670] (t=240003 jiffies g=50770253 c=50770252 q=4893) [1488104.993674] [] ? kthread+0xd1/0xe0 [1488104.993676] [] ? kthread+0xd1/0xe0 [1488104.993678] [] ? insert_kthread_work+0x40/0x40 [1488104.993679] [] ? insert_kthread_work+0x40/0x40 [1488104.993681] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.993684] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.993685] [] ? insert_kthread_work+0x40/0x40 [1488104.993687] [] ? insert_kthread_work+0x40/0x40 [1488104.993688] Task dump for CPU 19: [1488104.993688] Task dump for CPU 5: [1488104.993690] ptlrpcd_06_01 R [1488104.993691] ptlrpcd_05_02 R [1488104.993691] running task [1488104.993693] 0 54229 2 0x00000088 [1488104.993694] running task 0 54218 2 0x00000088 [1488104.993695] Call Trace: [1488104.993695] Call Trace: [1488104.993735] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.993771] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.993809] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.993810] [] ? del_timer_sync+0x52/0x60 [1488104.993848] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.993884] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.993924] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.993960] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.993962] [] ? wake_up_state+0x20/0x20 [1488104.993999] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.994038] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.994040] [] ? wake_up_state+0x20/0x20 [1488104.994042] [] ? kthread+0xd1/0xe0 [1488104.994080] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.994082] [] ? insert_kthread_work+0x40/0x40 [1488104.994083] [] ? kthread+0xd1/0xe0 [1488104.994085] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.994087] [] ? insert_kthread_work+0x40/0x40 [1488104.994089] [] ? insert_kthread_work+0x40/0x40 [1488104.994091] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.994092] Task dump for CPU 8: [1488104.994093] [] ? insert_kthread_work+0x40/0x40 [1488104.994095] Task dump for CPU 21: [1488104.994096] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488104.994097] Call Trace: [1488104.994136] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.994137] ptlrpcd_06_11 R running task 0 54239 2 0x00000088 [1488104.994175] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.994175] Call Trace: [1488104.994213] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.994249] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.994288] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.994325] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.994326] [] ? wake_up_state+0x20/0x20 [1488104.994363] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.994401] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.994439] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.994441] [] ? kthread+0xd1/0xe0 [1488104.994443] [] ? wake_up_state+0x20/0x20 [1488104.994444] [] ? insert_kthread_work+0x40/0x40 [1488104.994482] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.994484] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.994486] [] ? kthread+0xd1/0xe0 [1488104.994488] [] ? insert_kthread_work+0x40/0x40 [1488104.994490] [] ? insert_kthread_work+0x40/0x40 [1488104.994491] Task dump for CPU 13: [1488104.994493] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.994495] [] ? insert_kthread_work+0x40/0x40 [1488104.994496] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488104.994497] Task dump for CPU 24: [1488104.994498] Call Trace: [1488104.994537] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.994539] ptlrpcd_00_11 R running task 0 54167 2 0x00000088 [1488104.994577] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.994577] Call Trace: [1488104.994615] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.994651] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.994690] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.994692] [] ? del_timer_sync+0x52/0x60 [1488104.994693] [] ? wake_up_state+0x20/0x20 [1488104.994730] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.994769] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.994806] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.994807] [] ? kthread+0xd1/0xe0 [1488104.994845] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.994847] [] ? insert_kthread_work+0x40/0x40 [1488104.994849] [] ? wake_up_state+0x20/0x20 [1488104.994851] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.994889] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.994891] [] ? insert_kthread_work+0x40/0x40 [1488104.994893] [] ? kthread+0xd1/0xe0 [1488104.994894] Task dump for CPU 0: [1488104.994895] [] ? insert_kthread_work+0x40/0x40 [1488104.994898] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.994900] [] ? insert_kthread_work+0x40/0x40 [1488104.994901] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488104.994903] Task dump for CPU 26: [1488104.994903] Call Trace: [1488104.994943] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.994944] ptlrpcd_01_00 R running task 0 54168 2 0x00000088 [1488104.994984] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.994985] Call Trace: [1488104.995025] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.995062] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.995103] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.995140] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.995141] [] ? wake_up_state+0x20/0x20 [1488104.995178] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.995219] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.995257] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.995259] [] ? kthread+0xd1/0xe0 [1488104.995261] [] ? wake_up_state+0x20/0x20 [1488104.995263] [] ? insert_kthread_work+0x40/0x40 [1488104.995301] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.995303] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.995304] [] ? kthread+0xd1/0xe0 [1488104.995306] [] ? insert_kthread_work+0x40/0x40 [1488104.995307] [] ? insert_kthread_work+0x40/0x40 [1488104.995308] Task dump for CPU 5: [1488104.995310] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.995313] [] ? insert_kthread_work+0x40/0x40 [1488104.995314] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488104.995315] Task dump for CPU 32: [1488104.995316] Task dump for CPU 16: [1488104.995316] Call Trace: [1488104.995318] ptlrpcd_00_10 R [1488104.995354] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.995355] ptlrpcd_00_07 R [1488104.995356] running task [1488104.995393] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.995394] 0 54166 2 0x00000088 [1488104.995395] running task 0 54163 2 0x00000088 [1488104.995432] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.995432] Call Trace: [1488104.995433] Call Trace: [1488104.995471] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.995507] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.995545] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.995547] [] ? wake_up_state+0x20/0x20 [1488104.995583] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.995621] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.995659] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.995695] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.995732] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.995734] [] ? kthread+0xd1/0xe0 [1488104.995772] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.995811] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.995813] [] ? insert_kthread_work+0x40/0x40 [1488104.995815] [] ? wake_up_state+0x20/0x20 [1488104.995817] [] ? wake_up_state+0x20/0x20 [1488104.995819] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.995857] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.995896] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.995897] [] ? insert_kthread_work+0x40/0x40 [1488104.995899] [] ? kthread+0xd1/0xe0 [1488104.995900] [] ? kthread+0xd1/0xe0 [1488104.995901] Task dump for CPU 8: [1488104.995903] [] ? insert_kthread_work+0x40/0x40 [1488104.995905] [] ? insert_kthread_work+0x40/0x40 [1488104.995907] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.995909] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.995912] [] ? insert_kthread_work+0x40/0x40 [1488104.995913] [] ? insert_kthread_work+0x40/0x40 [1488104.995915] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488104.995916] Task dump for CPU 40: [1488104.995916] Task dump for CPU 19: [1488104.995917] Call Trace: [1488104.995918] ptlrpcd_00_09 R [1488104.995919] ptlrpcd_05_02 R [1488104.995919] running task [1488104.995920] [1488104.995921] 0 54165 2 0x00000088 [1488104.995923] [] sched_show_task+0xa8/0x110 [1488104.995925] running task 0 54218 2 0x00000088 [1488104.995925] Call Trace: [1488104.995927] [] dump_cpu_task+0x39/0x70 [1488104.995927] Call Trace: [1488104.995930] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488104.995967] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.995969] [] sched_show_task+0xa8/0x110 [1488104.995971] [] rcu_check_callbacks+0x442/0x730 [1488104.995972] [] ? del_timer_sync+0x52/0x60 [1488104.995975] [] dump_cpu_task+0x39/0x70 [1488104.995977] [] ? tick_sched_do_timer+0x50/0x50 [1488104.996013] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.996015] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488104.996017] [] update_process_times+0x46/0x80 [1488104.996053] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.996055] [] rcu_check_callbacks+0x442/0x730 [1488104.996057] [] tick_sched_handle+0x30/0x70 [1488104.996095] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.996097] [] ? tick_sched_do_timer+0x50/0x50 [1488104.996099] [] tick_sched_timer+0x39/0x80 [1488104.996100] [] ? wake_up_state+0x20/0x20 [1488104.996102] [] update_process_times+0x46/0x80 [1488104.996105] [] __hrtimer_run_queues+0xf3/0x270 [1488104.996142] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.996144] [] tick_sched_handle+0x30/0x70 [1488104.996147] [] hrtimer_interrupt+0xaf/0x1d0 [1488104.996148] [] ? kthread+0xd1/0xe0 [1488104.996150] [] tick_sched_timer+0x39/0x80 [1488104.996152] [] local_apic_timer_interrupt+0x3b/0x60 [1488104.996154] [] ? insert_kthread_work+0x40/0x40 [1488104.996156] [] __hrtimer_run_queues+0xf3/0x270 [1488104.996158] [] smp_apic_timer_interrupt+0x43/0x60 [1488104.996160] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.996162] [] hrtimer_interrupt+0xaf/0x1d0 [1488104.996164] [] apic_timer_interrupt+0x162/0x170 [1488104.996166] [] ? insert_kthread_work+0x40/0x40 [1488104.996168] [] local_apic_timer_interrupt+0x3b/0x60 [1488104.996170] Task dump for CPU 21: [1488104.996172] [] smp_apic_timer_interrupt+0x43/0x60 [1488104.996174] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [1488104.996177] [] apic_timer_interrupt+0x162/0x170 [1488104.996179] [] queued_spin_lock_slowpath+0xb/0xf [1488104.996180] ptlrpcd_06_11 R running task [1488104.996182] [] _raw_spin_lock+0x20/0x30 [1488104.996184] 0 54239 2 0x00000088 [1488104.996186] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488104.996194] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488104.996195] Call Trace: [1488104.996196] [] queued_spin_lock_slowpath+0xb/0xf [1488104.996208] [] LNetMDUnlink+0x78/0x180 [lnet] [1488104.996244] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.996247] [] _raw_spin_lock+0x20/0x30 [1488104.996284] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.996321] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.996328] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488104.996369] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.996406] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.996416] [] LNetMDUnlink+0x78/0x180 [lnet] [1488104.996457] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.996495] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.996531] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.996573] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.996575] [] ? wake_up_state+0x20/0x20 [1488104.996612] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.996614] [] ? wake_up_state+0x20/0x20 [1488104.996652] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.996688] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.996729] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.996731] [] ? kthread+0xd1/0xe0 [1488104.996769] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.996771] [] kthread+0xd1/0xe0 [1488104.996773] [] ? insert_kthread_work+0x40/0x40 [1488104.996775] [] ? wake_up_state+0x20/0x20 [1488104.996777] [] ? insert_kthread_work+0x40/0x40 [1488104.996779] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.996817] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.996819] [] ret_from_fork_nospec_begin+0xe/0x21 [1488104.996821] [] ? insert_kthread_work+0x40/0x40 [1488104.996822] [] kthread+0xd1/0xe0 [1488104.996824] [] ? insert_kthread_work+0x40/0x40 [1488104.996825] Task dump for CPU 24: [1488104.996827] [] ? insert_kthread_work+0x40/0x40 [1488104.996828] Task dump for CPU 13: [1488104.996830] [] ret_from_fork_nospec_begin+0xe/0x21 [1488104.996831] ptlrpcd_00_11 R [1488104.996832] ptlrpcd_06_02 R [1488104.996833] [] ? insert_kthread_work+0x40/0x40 [1488104.996834] running task [1488104.996835] 0 54167 2 0x00000088 [1488104.996835] Task dump for CPU 42: [1488104.996837] running task 0 54230 2 0x00000088 [1488104.996837] Call Trace: [1488104.996838] Call Trace: [1488104.996875] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.996916] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.996918] [] ? del_timer_sync+0x52/0x60 [1488104.996919] ptlrpcd_01_01 R running task 0 54169 2 0x00000088 [1488104.996960] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.996996] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.996997] Call Trace: [1488104.997037] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.997073] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.997110] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.997152] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.997190] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.997227] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.997228] [] ? wake_up_state+0x20/0x20 [1488104.997230] [] ? wake_up_state+0x20/0x20 [1488104.997267] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.997308] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.997346] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.997384] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.997385] [] ? kthread+0xd1/0xe0 [1488104.997387] [] ? kthread+0xd1/0xe0 [1488104.997389] [] ? wake_up_state+0x20/0x20 [1488104.997391] [] ? insert_kthread_work+0x40/0x40 [1488104.997393] [] ? insert_kthread_work+0x40/0x40 [1488104.997431] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.997432] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.997435] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.997436] [] ? kthread+0xd1/0xe0 [1488104.997438] [] ? insert_kthread_work+0x40/0x40 [1488104.997440] [] ? insert_kthread_work+0x40/0x40 [1488104.997443] [] ? insert_kthread_work+0x40/0x40 [1488104.997445] Task dump for CPU 0: [1488104.997447] Task dump for CPU 26: [1488104.997449] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.997454] ptlrpcd_00_08 R [1488104.997456] [] ? insert_kthread_work+0x40/0x40 [1488104.997457] ptlrpcd_01_00 R [1488104.997459] running task [1488104.997461] Task dump for CPU 43: [1488104.997464] 0 54164 2 0x00000088 [1488104.997467] running task 0 54168 2 0x00000088 [1488104.997469] Call Trace: [1488104.997471] Call Trace: [1488104.997511] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.997550] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.997552] ptlrpcd_05_10 R running task 0 54226 2 0x00000088 [1488104.997590] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.997629] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.997630] Call Trace: [1488104.997668] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.997707] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.997744] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.997783] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.997823] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.997861] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.997863] [] ? wake_up_state+0x20/0x20 [1488104.997866] [] ? wake_up_state+0x20/0x20 [1488104.997903] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.997942] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.997982] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.998020] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.998023] [] ? kthread+0xd1/0xe0 [1488104.998026] [] ? kthread+0xd1/0xe0 [1488104.998028] [] ? wake_up_state+0x20/0x20 [1488104.998031] [] ? insert_kthread_work+0x40/0x40 [1488104.998034] [] ? insert_kthread_work+0x40/0x40 [1488104.998072] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.998075] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.998078] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.998080] [] ? kthread+0xd1/0xe0 [1488104.998083] [] ? insert_kthread_work+0x40/0x40 [1488104.998085] [] ? insert_kthread_work+0x40/0x40 [1488104.998088] [] ? insert_kthread_work+0x40/0x40 [1488104.998090] Task dump for CPU 5: [1488104.998092] Task dump for CPU 16: [1488104.998095] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.998098] ptlrpcd_06_01 R [1488104.998101] [] ? insert_kthread_work+0x40/0x40 [1488104.998102] ptlrpcd_00_07 R [1488104.998104] running task [1488104.998105] Task dump for CPU 47: [1488104.998108] 0 54229 2 0x00000088 [1488104.998111] running task 0 54163 2 0x00000088 [1488104.998113] Call Trace: [1488104.998115] Call Trace: [1488104.998156] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.998194] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.998196] ptlrpcd_07_02 R running task 0 54242 2 0x00000088 [1488104.998235] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.998276] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.998277] Call Trace: [1488104.998314] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.998356] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.998393] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.998431] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.998474] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.998511] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.998514] [] ? wake_up_state+0x20/0x20 [1488104.998517] [] ? wake_up_state+0x20/0x20 [1488104.998554] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.998593] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.998633] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.998672] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.998674] [] ? kthread+0xd1/0xe0 [1488104.998677] [] ? kthread+0xd1/0xe0 [1488104.998679] [] ? wake_up_state+0x20/0x20 [1488104.998682] [] ? insert_kthread_work+0x40/0x40 [1488104.998685] [] ? insert_kthread_work+0x40/0x40 [1488104.998723] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.998726] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.998730] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.998731] [] ? kthread+0xd1/0xe0 [1488104.998735] [] ? insert_kthread_work+0x40/0x40 [1488104.998737] [] ? insert_kthread_work+0x40/0x40 [1488104.998740] [] ? insert_kthread_work+0x40/0x40 [1488104.998742] Task dump for CPU 8: [1488104.998744] Task dump for CPU 19: [1488104.998746] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.998750] ptlrpcd_00_02 R [1488104.998753] [] ? insert_kthread_work+0x40/0x40 [1488104.998754] ptlrpcd_05_02 R [1488104.998756] running task [1488104.998757] Task dump for CPU 48: [1488104.998759] Task dump for CPU 32: [1488104.998761] 0 54158 2 0x00000088 [1488104.998764] running task 0 54218 2 0x00000088 [1488104.998766] ptlrpcd_00_05 R [1488104.998767] Call Trace: [1488104.998769] Call Trace: [1488104.998770] ptlrpcd_00_10 R [1488104.998772] running task [1488104.998810] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.998847] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.998849] 0 54161 2 0x00000088 [1488104.998851] running task 0 54166 2 0x00000088 [1488104.998888] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.998891] [] ? del_timer_sync+0x52/0x60 [1488104.998892] Call Trace: [1488104.998892] Call Trace: [1488104.998930] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.998969] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.999005] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.999042] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.999081] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.999119] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.999156] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.999193] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.999196] [] ? wake_up_state+0x20/0x20 [1488104.999236] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.999273] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.999310] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.999348] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.999352] [] ? wake_up_state+0x20/0x20 [1488104.999390] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.999428] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.999431] [] ? kthread+0xd1/0xe0 [1488104.999470] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.999472] [] ? wake_up_state+0x20/0x20 [1488104.999475] [] ? wake_up_state+0x20/0x20 [1488104.999477] [] ? insert_kthread_work+0x40/0x40 [1488104.999480] [] ? kthread+0xd1/0xe0 [1488104.999518] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.999556] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488104.999560] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.999562] [] ? insert_kthread_work+0x40/0x40 [1488104.999564] [] ? kthread+0xd1/0xe0 [1488104.999566] [] ? kthread+0xd1/0xe0 [1488104.999569] [] ? insert_kthread_work+0x40/0x40 [1488104.999572] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.999575] [] ? insert_kthread_work+0x40/0x40 [1488104.999577] [] ? insert_kthread_work+0x40/0x40 [1488104.999579] Task dump for CPU 13: [1488104.999582] [] ? insert_kthread_work+0x40/0x40 [1488104.999584] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.999587] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488104.999591] Task dump for CPU 21: [1488104.999592] [] ? insert_kthread_work+0x40/0x40 [1488104.999595] [] ? insert_kthread_work+0x40/0x40 [1488104.999599] ptlrpcd_06_02 R running task [1488104.999599] Task dump for CPU 56: [1488104.999601] Task dump for CPU 40: [1488104.999603] 0 54230 2 0x00000088 [1488104.999606] ptlrpcd_06_11 R running task [1488104.999607] ptlrpcd_00_03 R [1488104.999609] Call Trace: [1488104.999612] 0 54239 2 0x00000088 [1488104.999613] ptlrpcd_00_09 R [1488104.999614] running task [1488104.999652] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.999653] Call Trace: [1488104.999656] 0 54159 2 0x00000088 [1488104.999658] running task 0 54165 2 0x00000088 [1488104.999696] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.999733] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.999734] Call Trace: [1488104.999735] Call Trace: [1488104.999773] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488104.999811] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.999852] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488104.999889] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488104.999928] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488104.999966] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.000006] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.000043] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.000046] [] ? wake_up_state+0x20/0x20 [1488105.000083] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.000125] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.000163] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.000201] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.000241] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.000242] [] ? wake_up_state+0x20/0x20 [1488105.000281] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.000283] [] ? kthread+0xd1/0xe0 [1488105.000286] [] ? wake_up_state+0x20/0x20 [1488105.000325] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.000328] [] ? wake_up_state+0x20/0x20 [1488105.000331] [] ? insert_kthread_work+0x40/0x40 [1488105.000370] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.000371] [] ? kthread+0xd1/0xe0 [1488105.000410] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.000413] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.000416] [] ? kthread+0xd1/0xe0 [1488105.000418] [] ? insert_kthread_work+0x40/0x40 [1488105.000420] [] ? kthread+0xd1/0xe0 [1488105.000423] [] ? insert_kthread_work+0x40/0x40 [1488105.000426] [] ? insert_kthread_work+0x40/0x40 [1488105.000429] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.000431] [] ? insert_kthread_work+0x40/0x40 [1488105.000434] Task dump for CPU 0: [1488105.000438] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.000440] [] ? insert_kthread_work+0x40/0x40 [1488105.000442] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.000448] [] ? insert_kthread_work+0x40/0x40 [1488105.000449] Task dump for CPU 24: [1488105.000451] [] ? insert_kthread_work+0x40/0x40 [1488105.000456] Task dump for CPU 61: [1488105.000456] ptlrpcd_00_08 R running task [1488105.000457] Task dump for CPU 42: [1488105.000460] 0 54164 2 0x00000088 [1488105.000463] ptlrpcd_00_11 R [1488105.000465] ptlrpcd_06_08 R [1488105.000465] running task [1488105.000468] Call Trace: [1488105.000471] ptlrpcd_01_01 R [1488105.000473] 0 54167 2 0x00000088 [1488105.000474] running task [1488105.000514] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.000517] 0 54236 2 0x00000088 [1488105.000518] Call Trace: [1488105.000521] running task 0 54169 2 0x00000088 [1488105.000561] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.000562] Call Trace: [1488105.000604] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.000605] Call Trace: [1488105.000644] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.000683] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.000685] [] ? del_timer_sync+0x52/0x60 [1488105.000723] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.000763] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.000766] [] ? del_timer_sync+0x52/0x60 [1488105.000808] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.000846] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.000850] [] ? wake_up_state+0x20/0x20 [1488105.000889] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.000929] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.000967] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.001007] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.001046] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.001088] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.001127] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.001130] [] ? kthread+0xd1/0xe0 [1488105.001170] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.001172] [] ? wake_up_state+0x20/0x20 [1488105.001174] [] ? wake_up_state+0x20/0x20 [1488105.001178] [] ? insert_kthread_work+0x40/0x40 [1488105.001182] [] ? wake_up_state+0x20/0x20 [1488105.001224] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.001262] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.001266] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.001306] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.001308] [] ? kthread+0xd1/0xe0 [1488105.001310] [] ? kthread+0xd1/0xe0 [1488105.001315] [] ? insert_kthread_work+0x40/0x40 [1488105.001318] [] ? kthread+0xd1/0xe0 [1488105.001320] [] ? insert_kthread_work+0x40/0x40 [1488105.001323] [] ? insert_kthread_work+0x40/0x40 [1488105.001325] Task dump for CPU 5: [1488105.001329] [] ? insert_kthread_work+0x40/0x40 [1488105.001332] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.001335] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.001341] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.001343] [] ? insert_kthread_work+0x40/0x40 [1488105.001346] [] ? insert_kthread_work+0x40/0x40 [1488105.001351] [] ? insert_kthread_work+0x40/0x40 [1488105.001352] Task dump for CPU 26: [1488105.001353] Task dump for CPU 43: [1488105.001357] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.001360] Task dump for CPU 64: [1488105.001362] ptlrpcd_01_00 R [1488105.001364] Call Trace: [1488105.001367] ptlrpcd_05_10 R [1488105.001369] ptlrpcd_00_06 R [1488105.001369] running task [1488105.001408] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.001410] running task [1488105.001412] 0 54168 2 0x00000088 [1488105.001414] 0 54226 2 0x00000088 [1488105.001453] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.001456] running task 0 54162 2 0x00000088 [1488105.001457] Call Trace: [1488105.001458] Call Trace: [1488105.001496] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.001499] Call Trace: [1488105.001540] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.001578] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.001619] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.001658] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.001698] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.001737] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.001740] [] ? wake_up_state+0x20/0x20 [1488105.001779] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.001820] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.001858] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.001898] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.001937] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.001979] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002018] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002020] [] ? kthread+0xd1/0xe0 [1488105.002060] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002062] [] ? wake_up_state+0x20/0x20 [1488105.002066] [] ? wake_up_state+0x20/0x20 [1488105.002070] [] ? insert_kthread_work+0x40/0x40 [1488105.002073] [] ? wake_up_state+0x20/0x20 [1488105.002115] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.002153] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.002157] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.002197] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.002199] [] ? kthread+0xd1/0xe0 [1488105.002201] [] ? kthread+0xd1/0xe0 [1488105.002205] [] ? insert_kthread_work+0x40/0x40 [1488105.002209] [] ? kthread+0xd1/0xe0 [1488105.002211] [] ? insert_kthread_work+0x40/0x40 [1488105.002213] [] ? insert_kthread_work+0x40/0x40 [1488105.002216] Task dump for CPU 8: [1488105.002220] [] ? insert_kthread_work+0x40/0x40 [1488105.002223] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.002226] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.002232] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.002234] [] ? insert_kthread_work+0x40/0x40 [1488105.002237] [] ? insert_kthread_work+0x40/0x40 [1488105.002243] [] ? insert_kthread_work+0x40/0x40 [1488105.002245] Task dump for CPU 16: [1488105.002248] Task dump for CPU 47: [1488105.002251] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.002254] Task dump for CPU 69: [1488105.002258] ptlrpcd_00_07 R [1488105.002261] Call Trace: [1488105.002264] ptlrpcd_07_02 R [1488105.002268] ptlrpcd_06_06 R [1488105.002269] running task [1488105.002309] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.002311] running task [1488105.002314] 0 54163 2 0x00000088 [1488105.002317] 0 54242 2 0x00000088 [1488105.002356] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.002360] running task 0 54234 2 0x00000088 [1488105.002361] Call Trace: [1488105.002363] Call Trace: [1488105.002403] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.002404] Call Trace: [1488105.002443] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.002482] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.002522] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002562] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.002600] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.002639] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.002643] [] ? wake_up_state+0x20/0x20 [1488105.002682] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.002719] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.002758] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.002798] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.002837] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.002876] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002916] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002919] [] ? kthread+0xd1/0xe0 [1488105.002959] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.002962] [] ? wake_up_state+0x20/0x20 [1488105.002965] [] ? wake_up_state+0x20/0x20 [1488105.002968] [] ? insert_kthread_work+0x40/0x40 [1488105.002972] [] ? wake_up_state+0x20/0x20 [1488105.003011] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.003052] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.003056] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.003096] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.003098] [] ? kthread+0xd1/0xe0 [1488105.003101] [] ? kthread+0xd1/0xe0 [1488105.003105] [] ? insert_kthread_work+0x40/0x40 [1488105.003108] [] ? kthread+0xd1/0xe0 [1488105.003111] [] ? insert_kthread_work+0x40/0x40 [1488105.003114] [] ? insert_kthread_work+0x40/0x40 [1488105.003117] Task dump for CPU 13: [1488105.003121] [] ? insert_kthread_work+0x40/0x40 [1488105.003125] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.003128] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.003135] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.003138] [] ? insert_kthread_work+0x40/0x40 [1488105.003141] [] ? insert_kthread_work+0x40/0x40 [1488105.003148] [] ? insert_kthread_work+0x40/0x40 [1488105.003149] Task dump for CPU 19: [1488105.003152] Task dump for CPU 48: [1488105.003153] Task dump for CPU 32: [1488105.003156] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.003159] Task dump for CPU 72: [1488105.003163] ptlrpcd_05_02 R [1488105.003164] ptlrpcd_00_05 R [1488105.003166] Call Trace: [1488105.003169] ptlrpcd_00_10 R [1488105.003172] ptlrpcd_00_04 R [1488105.003174] running task [1488105.003175] running task [1488105.003214] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.003216] running task [1488105.003220] 0 54218 2 0x00000088 [1488105.003222] 0 54161 2 0x00000088 [1488105.003223] 0 54166 2 0x00000088 [1488105.003263] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.003266] running task 0 54160 2 0x00000088 [1488105.003268] Call Trace: [1488105.003269] Call Trace: [1488105.003271] Call Trace: [1488105.003309] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.003310] Call Trace: [1488105.003348] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.003389] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.003427] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.003466] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.003506] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.003508] [] ? del_timer_sync+0x52/0x60 [1488105.003548] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.003585] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.003589] [] ? wake_up_state+0x20/0x20 [1488105.003628] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.003665] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.003707] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.003744] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.003784] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.003822] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.003859] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.003900] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.003938] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.003941] [] ? kthread+0xd1/0xe0 [1488105.003982] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.004021] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.004024] [] ? wake_up_state+0x20/0x20 [1488105.004026] [] ? wake_up_state+0x20/0x20 [1488105.004030] [] ? insert_kthread_work+0x40/0x40 [1488105.004033] [] ? wake_up_state+0x20/0x20 [1488105.004036] [] ? wake_up_state+0x20/0x20 [1488105.004077] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.004115] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.004119] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.004159] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.004198] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.004201] [] ? kthread+0xd1/0xe0 [1488105.004203] [] ? kthread+0xd1/0xe0 [1488105.004207] [] ? insert_kthread_work+0x40/0x40 [1488105.004210] [] ? kthread+0xd1/0xe0 [1488105.004213] [] ? kthread+0xd1/0xe0 [1488105.004216] [] ? insert_kthread_work+0x40/0x40 [1488105.004218] [] ? insert_kthread_work+0x40/0x40 [1488105.004219] Task dump for CPU 0: [1488105.004222] [] ? insert_kthread_work+0x40/0x40 [1488105.004225] [] ? insert_kthread_work+0x40/0x40 [1488105.004229] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.004231] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.004235] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.004239] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.004242] [] ? insert_kthread_work+0x40/0x40 [1488105.004244] [] ? insert_kthread_work+0x40/0x40 [1488105.004248] [] ? insert_kthread_work+0x40/0x40 [1488105.004250] [] ? insert_kthread_work+0x40/0x40 [1488105.004253] Task dump for CPU 56: [1488105.004253] Task dump for CPU 40: [1488105.004256] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.004258] Task dump for CPU 79: [1488105.004260] Task dump for CPU 21: [1488105.004263] ptlrpcd_00_03 R [1488105.004265] Call Trace: [1488105.004266] ptlrpcd_00_09 R [1488105.004269] ptlrpcd_07_00 R [1488105.004270] ptlrpcd_06_11 R [1488105.004271] running task [1488105.004310] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.004311] running task [1488105.004314] running task [1488105.004316] 0 54159 2 0x00000088 [1488105.004317] 0 54165 2 0x00000088 [1488105.004355] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.004357] 0 54240 2 0x00000088 [1488105.004360] running task 0 54239 2 0x00000088 [1488105.004362] Call Trace: [1488105.004362] Call Trace: [1488105.004400] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.004401] Call Trace: [1488105.004402] Call Trace: [1488105.004442] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.004482] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.004520] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.004557] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.004561] [] sched_show_task+0xa8/0x110 [1488105.004598] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.004602] [] ? wake_up_state+0x20/0x20 [1488105.004639] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.004677] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.004680] [] dump_cpu_task+0x39/0x70 [1488105.004717] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.004755] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.004792] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.004830] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.004833] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.004871] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.004874] [] ? kthread+0xd1/0xe0 [1488105.004913] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.004952] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.004955] [] rcu_check_callbacks+0x442/0x730 [1488105.004957] [] ? wake_up_state+0x20/0x20 [1488105.004959] [] ? insert_kthread_work+0x40/0x40 [1488105.004962] [] ? wake_up_state+0x20/0x20 [1488105.004965] [] ? wake_up_state+0x20/0x20 [1488105.004968] [] ? tick_sched_do_timer+0x50/0x50 [1488105.005007] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.005009] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.005048] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.005087] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.005090] [] update_process_times+0x46/0x80 [1488105.005092] [] ? kthread+0xd1/0xe0 [1488105.005095] [] ? insert_kthread_work+0x40/0x40 [1488105.005097] [] ? kthread+0xd1/0xe0 [1488105.005100] [] ? kthread+0xd1/0xe0 [1488105.005103] [] tick_sched_handle+0x30/0x70 [1488105.005105] [] ? insert_kthread_work+0x40/0x40 [1488105.005106] Task dump for CPU 5: [1488105.005109] [] ? insert_kthread_work+0x40/0x40 [1488105.005112] [] ? insert_kthread_work+0x40/0x40 [1488105.005115] [] tick_sched_timer+0x39/0x80 [1488105.005117] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.005122] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.005125] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.005128] [] __hrtimer_run_queues+0xf3/0x270 [1488105.005131] [] ? insert_kthread_work+0x40/0x40 [1488105.005134] [] ? insert_kthread_work+0x40/0x40 [1488105.005137] [] ? insert_kthread_work+0x40/0x40 [1488105.005140] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.005142] Task dump for CPU 42: [1488105.005144] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.005146] Task dump for CPU 80: [1488105.005148] Task dump for CPU 24: [1488105.005151] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.005154] Call Trace: [1488105.005156] ptlrpcd_01_01 R [1488105.005158] ptlrpcd_00_01 R [1488105.005161] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.005162] ptlrpcd_00_11 R [1488105.005200] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.005201] running task [1488105.005203] running task [1488105.005207] [] apic_timer_interrupt+0x162/0x170 [1488105.005208] 0 54169 2 0x00000088 [1488105.005246] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.005248] 0 54157 2 0x00000088 [1488105.005251] running task 0 54167 2 0x00000088 [1488105.005253] Call Trace: [1488105.005290] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.005292] Call Trace: [1488105.005293] Call Trace: [1488105.005297] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.005334] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.005373] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.005411] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.005449] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.005452] [] queued_spin_lock_slowpath+0xb/0xf [1488105.005489] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.005492] [] ? wake_up_state+0x20/0x20 [1488105.005529] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.005532] [] ? del_timer_sync+0x52/0x60 [1488105.005536] [] _raw_spin_lock+0x20/0x30 [1488105.005573] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.005612] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.005649] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.005686] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.005696] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.005734] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.005736] [] ? kthread+0xd1/0xe0 [1488105.005775] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.005813] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.005825] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.005827] [] ? wake_up_state+0x20/0x20 [1488105.005830] [] ? insert_kthread_work+0x40/0x40 [1488105.005833] [] ? wake_up_state+0x20/0x20 [1488105.005871] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.005910] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.005949] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.005952] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.005990] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.005993] [] ? wake_up_state+0x20/0x20 [1488105.006032] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.006034] [] ? kthread+0xd1/0xe0 [1488105.006037] [] ? insert_kthread_work+0x40/0x40 [1488105.006040] [] ? kthread+0xd1/0xe0 [1488105.006079] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.006118] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.006120] [] ? insert_kthread_work+0x40/0x40 [1488105.006121] Task dump for CPU 8: [1488105.006124] [] ? insert_kthread_work+0x40/0x40 [1488105.006127] [] ? kthread+0xd1/0xe0 [1488105.006167] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.006169] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.006174] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.006177] [] ? insert_kthread_work+0x40/0x40 [1488105.006180] [] ? wake_up_state+0x20/0x20 [1488105.006182] [] ? insert_kthread_work+0x40/0x40 [1488105.006186] [] ? insert_kthread_work+0x40/0x40 [1488105.006189] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.006229] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.006230] Task dump for CPU 43: [1488105.006232] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.006235] Task dump for CPU 88: [1488105.006237] [] ? insert_kthread_work+0x40/0x40 [1488105.006240] [] kthread+0xd1/0xe0 [1488105.006243] Call Trace: [1488105.006244] ptlrpcd_05_10 R [1488105.006247] Task dump for CPU 26: [1488105.006250] [] ? insert_kthread_work+0x40/0x40 [1488105.006250] ptlrpcd_00_00 R [1488105.006288] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.006289] running task [1488105.006292] running task [1488105.006296] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.006297] 0 54226 2 0x00000088 [1488105.006334] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.006336] 0 54156 2 0x00000088 [1488105.006340] [] ? insert_kthread_work+0x40/0x40 [1488105.006341] Call Trace: [1488105.006378] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.006379] Call Trace: [1488105.006382] ptlrpcd_01_00 R running task 0 54168 2 0x00000088 [1488105.006384] Task dump for CPU 61: [1488105.006420] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.006459] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.006497] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.006498] Call Trace: [1488105.006538] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.006541] [] ? wake_up_state+0x20/0x20 [1488105.006578] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.006580] ptlrpcd_06_08 R [1488105.006582] [1488105.006619] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.006658] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.006696] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.006698] [] sched_show_task+0xa8/0x110 [1488105.006701] running task 0 54236 2 0x00000088 [1488105.006739] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.006742] [] ? kthread+0xd1/0xe0 [1488105.006781] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.006784] [] dump_cpu_task+0x39/0x70 [1488105.006786] Call Trace: [1488105.006788] [] ? wake_up_state+0x20/0x20 [1488105.006791] [] ? insert_kthread_work+0x40/0x40 [1488105.006793] [] ? wake_up_state+0x20/0x20 [1488105.006796] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.006836] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.006874] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.006877] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.006915] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.006919] [] rcu_check_callbacks+0x442/0x730 [1488105.006921] [] ? del_timer_sync+0x52/0x60 [1488105.006923] [] ? kthread+0xd1/0xe0 [1488105.006926] [] ? insert_kthread_work+0x40/0x40 [1488105.006928] [] ? kthread+0xd1/0xe0 [1488105.006932] [] ? tick_sched_do_timer+0x50/0x50 [1488105.006971] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.006972] [] ? insert_kthread_work+0x40/0x40 [1488105.006974] Task dump for CPU 13: [1488105.006977] [] ? insert_kthread_work+0x40/0x40 [1488105.006980] [] update_process_times+0x46/0x80 [1488105.007018] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.007020] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.007025] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.007028] [] tick_sched_handle+0x30/0x70 [1488105.007068] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.007070] [] ? insert_kthread_work+0x40/0x40 [1488105.007074] [] ? insert_kthread_work+0x40/0x40 [1488105.007077] [] tick_sched_timer+0x39/0x80 [1488105.007080] [] ? wake_up_state+0x20/0x20 [1488105.007081] Task dump for CPU 47: [1488105.007084] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.007086] Task dump for CPU 95: [1488105.007089] [] __hrtimer_run_queues+0xf3/0x270 [1488105.007132] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.007134] Call Trace: [1488105.007136] ptlrpcd_07_02 R [1488105.007140] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.007142] [] ? kthread+0xd1/0xe0 [1488105.007143] ptlrpcd_07_04 R [1488105.007181] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.007182] running task [1488105.007187] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.007190] [] ? insert_kthread_work+0x40/0x40 [1488105.007191] 0 54242 2 0x00000088 [1488105.007229] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.007231] running task 0 54244 2 0x00000088 [1488105.007234] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.007238] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.007238] Call Trace: [1488105.007276] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.007278] Call Trace: [1488105.007282] [] apic_timer_interrupt+0x162/0x170 [1488105.007285] [] ? insert_kthread_work+0x40/0x40 [1488105.007322] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.007361] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.007399] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.007403] Task dump for CPU 64: [1488105.007440] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.007442] [] ? wake_up_state+0x20/0x20 [1488105.007480] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.007484] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.007522] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.007560] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.007597] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.007600] [] queued_spin_lock_slowpath+0xb/0xf [1488105.007640] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.007642] [] ? kthread+0xd1/0xe0 [1488105.007681] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.007684] [] _raw_spin_lock+0x20/0x30 [1488105.007687] ptlrpcd_00_06 R running task 0 54162 2 0x00000088 [1488105.007689] [] ? wake_up_state+0x20/0x20 [1488105.007692] [] ? insert_kthread_work+0x40/0x40 [1488105.007694] [] ? wake_up_state+0x20/0x20 [1488105.007703] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.007705] Call Trace: [1488105.007744] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.007746] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.007785] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.007796] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.007835] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.007837] [] ? kthread+0xd1/0xe0 [1488105.007840] [] ? insert_kthread_work+0x40/0x40 [1488105.007842] [] ? kthread+0xd1/0xe0 [1488105.007881] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.007919] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.007922] [] ? insert_kthread_work+0x40/0x40 [1488105.007923] Task dump for CPU 0: [1488105.007926] [] ? insert_kthread_work+0x40/0x40 [1488105.007963] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.008002] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.008004] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.008009] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.008046] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.008086] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.008088] [] ? insert_kthread_work+0x40/0x40 [1488105.008092] [] ? insert_kthread_work+0x40/0x40 [1488105.008131] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.008134] [] ? wake_up_state+0x20/0x20 [1488105.008135] Task dump for CPU 48: [1488105.008138] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.008141] [] ? wake_up_state+0x20/0x20 [1488105.008184] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.008186] Call Trace: [1488105.008226] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.008229] [] ? kthread+0xd1/0xe0 [1488105.008232] ptlrpcd_00_05 R running task [1488105.008235] [] kthread+0xd1/0xe0 [1488105.008238] [] ? insert_kthread_work+0x40/0x40 [1488105.008239] 0 54161 2 0x00000088 [1488105.008243] [] sched_show_task+0xa8/0x110 [1488105.008246] [] ? insert_kthread_work+0x40/0x40 [1488105.008249] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.008250] Call Trace: [1488105.008253] [] dump_cpu_task+0x39/0x70 [1488105.008256] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.008259] [] ? insert_kthread_work+0x40/0x40 [1488105.008297] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.008299] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.008303] [] ? insert_kthread_work+0x40/0x40 [1488105.008304] Task dump for CPU 69: [1488105.008343] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.008346] [] rcu_check_callbacks+0x442/0x730 [1488105.008348] Task dump for CPU 32: [1488105.008353] Task dump for CPU 16: [1488105.008392] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.008395] [] ? tick_sched_do_timer+0x50/0x50 [1488105.008397] ptlrpcd_06_06 R [1488105.008399] ptlrpcd_00_10 R [1488105.008402] running task [1488105.008443] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.008445] [] update_process_times+0x46/0x80 [1488105.008447] ptlrpcd_00_07 R [1488105.008450] 0 54234 2 0x00000088 [1488105.008452] running task [1488105.008457] [] ? wake_up_state+0x20/0x20 [1488105.008459] [] tick_sched_handle+0x30/0x70 [1488105.008462] 0 54166 2 0x00000088 [1488105.008463] Call Trace: [1488105.008467] running task 0 54163 2 0x00000088 [1488105.008507] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.008509] [] tick_sched_timer+0x39/0x80 [1488105.008511] Call Trace: [1488105.008549] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.008551] Call Trace: [1488105.008555] [] ? kthread+0xd1/0xe0 [1488105.008559] [] __hrtimer_run_queues+0xf3/0x270 [1488105.008597] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.008634] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.008674] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.008678] [] ? insert_kthread_work+0x40/0x40 [1488105.008681] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.008719] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.008756] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.008795] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.008799] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.008802] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.008840] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.008879] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.008917] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.008921] [] ? insert_kthread_work+0x40/0x40 [1488105.008924] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.008963] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.008966] [] ? wake_up_state+0x20/0x20 [1488105.009006] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.009009] Task dump for CPU 56: [1488105.009012] [] apic_timer_interrupt+0x162/0x170 [1488105.009015] [] ? wake_up_state+0x20/0x20 [1488105.009054] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.009058] [] ? wake_up_state+0x20/0x20 [1488105.009062] ptlrpcd_00_03 R [1488105.009102] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.009105] [] ? kthread+0xd1/0xe0 [1488105.009145] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.009147] [1488105.009151] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [1488105.009154] [] ? kthread+0xd1/0xe0 [1488105.009157] [] ? insert_kthread_work+0x40/0x40 [1488105.009160] [] ? kthread+0xd1/0xe0 [1488105.009164] running task 0 54159 2 0x00000088 [1488105.009167] [] queued_spin_lock_slowpath+0xb/0xf [1488105.009170] [] ? insert_kthread_work+0x40/0x40 [1488105.009173] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.009177] [] ? insert_kthread_work+0x40/0x40 [1488105.009179] Call Trace: [1488105.009182] [] _raw_spin_lock+0x20/0x30 [1488105.009185] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.009188] [] ? insert_kthread_work+0x40/0x40 [1488105.009192] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.009231] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.009239] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.009243] [] ? insert_kthread_work+0x40/0x40 [1488105.009245] Task dump for CPU 72: [1488105.009249] [] ? insert_kthread_work+0x40/0x40 [1488105.009288] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.009299] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.009301] Task dump for CPU 40: [1488105.009306] Task dump for CPU 19: [1488105.009345] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.009383] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.009385] ptlrpcd_00_04 R [1488105.009387] ptlrpcd_00_09 R [1488105.009390] running task [1488105.009430] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.009468] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.009470] ptlrpcd_05_02 R [1488105.009473] 0 54160 2 0x00000088 [1488105.009475] running task [1488105.009480] [] ? wake_up_state+0x20/0x20 [1488105.009517] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.009520] 0 54165 2 0x00000088 [1488105.009521] Call Trace: [1488105.009525] running task 0 54218 2 0x00000088 [1488105.009566] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.009605] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.009606] Call Trace: [1488105.009644] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.009646] Call Trace: [1488105.009650] [] ? kthread+0xd1/0xe0 [1488105.009654] [] ? wake_up_state+0x20/0x20 [1488105.009692] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.009730] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.009768] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.009772] [] ? insert_kthread_work+0x40/0x40 [1488105.009811] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.009848] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.009886] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.009889] [] ? del_timer_sync+0x52/0x60 [1488105.009893] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.009896] [] kthread+0xd1/0xe0 [1488105.009933] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.009972] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.010010] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.010014] [] ? insert_kthread_work+0x40/0x40 [1488105.010017] [] ? insert_kthread_work+0x40/0x40 [1488105.010056] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.010059] [] ? wake_up_state+0x20/0x20 [1488105.010097] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.010099] Task dump for CPU 61: [1488105.010102] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.010105] [] ? wake_up_state+0x20/0x20 [1488105.010144] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.010185] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.010190] [] ? insert_kthread_work+0x40/0x40 [1488105.010229] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.010232] [] ? kthread+0xd1/0xe0 [1488105.010236] [] ? wake_up_state+0x20/0x20 [1488105.010239] Task dump for CPU 5: [1488105.010242] [] ? kthread+0xd1/0xe0 [1488105.010245] [] ? insert_kthread_work+0x40/0x40 [1488105.010286] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.010289] ptlrpcd_06_08 R running task 0 54236 2 0x00000088 [1488105.010293] [] ? insert_kthread_work+0x40/0x40 [1488105.010297] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.010301] [] ? kthread+0xd1/0xe0 [1488105.010303] Call Trace: [1488105.010308] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.010311] [] ? insert_kthread_work+0x40/0x40 [1488105.010314] [] ? insert_kthread_work+0x40/0x40 [1488105.010353] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.010355] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.010358] [] ? insert_kthread_work+0x40/0x40 [1488105.010360] Task dump for CPU 79: [1488105.010365] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.010369] [] ? del_timer_sync+0x52/0x60 [1488105.010370] Call Trace: [1488105.010372] Task dump for CPU 42: [1488105.010378] [] ? insert_kthread_work+0x40/0x40 [1488105.010417] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.010454] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.010456] ptlrpcd_07_00 R [1488105.010458] ptlrpcd_01_01 R [1488105.010462] Task dump for CPU 21: [1488105.010501] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.010539] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.010540] running task [1488105.010543] 0 54240 2 0x00000088 [1488105.010546] running task [1488105.010587] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.010624] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.010626] 0 54169 2 0x00000088 [1488105.010628] Call Trace: [1488105.010633] [] ? wake_up_state+0x20/0x20 [1488105.010672] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.010673] Call Trace: [1488105.010711] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.010714] ptlrpcd_06_11 R running task 0 54239 2 0x00000088 [1488105.010755] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.010758] [] ? wake_up_state+0x20/0x20 [1488105.010796] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.010834] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.010837] Call Trace: [1488105.010841] [] ? kthread+0xd1/0xe0 [1488105.010881] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.010918] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.010956] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.010994] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.010998] [] ? insert_kthread_work+0x40/0x40 [1488105.011001] [] ? kthread+0xd1/0xe0 [1488105.011039] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.011078] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.011116] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.011121] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.011124] [] ? insert_kthread_work+0x40/0x40 [1488105.011164] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.011166] [] ? wake_up_state+0x20/0x20 [1488105.011205] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.011208] [] ? insert_kthread_work+0x40/0x40 [1488105.011211] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.011215] [] ? wake_up_state+0x20/0x20 [1488105.011257] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.011297] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.011300] [] ? insert_kthread_work+0x40/0x40 [1488105.011339] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.011342] [] ? kthread+0xd1/0xe0 [1488105.011346] [] ? wake_up_state+0x20/0x20 [1488105.011348] Task dump for CPU 8: [1488105.011351] [] ? kthread+0xd1/0xe0 [1488105.011355] [] ? insert_kthread_work+0x40/0x40 [1488105.011395] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.011400] [] ? insert_kthread_work+0x40/0x40 [1488105.011404] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.011408] [] ? kthread+0xd1/0xe0 [1488105.011412] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.011415] [] ? insert_kthread_work+0x40/0x40 [1488105.011419] [] ? insert_kthread_work+0x40/0x40 [1488105.011422] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.011426] [] ? insert_kthread_work+0x40/0x40 [1488105.011428] Task dump for CPU 80: [1488105.011429] Task dump for CPU 64: [1488105.011433] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.011436] Call Trace: [1488105.011439] Task dump for CPU 43: [1488105.011443] ptlrpcd_00_01 R [1488105.011447] [] ? insert_kthread_work+0x40/0x40 [1488105.011486] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.011487] ptlrpcd_00_06 R [1488105.011489] ptlrpcd_05_10 R [1488105.011490] running task [1488105.011493] Task dump for CPU 24: [1488105.011531] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.011533] running task [1488105.011537] 0 54157 2 0x00000088 [1488105.011538] 0 54162 2 0x00000088 [1488105.011541] running task [1488105.011580] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.011582] 0 54226 2 0x00000088 [1488105.011584] Call Trace: [1488105.011585] Call Trace: [1488105.011627] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.011628] Call Trace: [1488105.011668] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.011706] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.011709] ptlrpcd_00_11 R running task 0 54167 2 0x00000088 [1488105.011713] [] ? wake_up_state+0x20/0x20 [1488105.011751] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.011791] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.011828] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.011830] Call Trace: [1488105.011870] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.011908] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.011950] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.011986] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.012025] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.012028] [] ? kthread+0xd1/0xe0 [1488105.012067] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.012108] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.012147] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.012150] [] ? del_timer_sync+0x52/0x60 [1488105.012154] [] ? insert_kthread_work+0x40/0x40 [1488105.012193] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.012196] [] ? wake_up_state+0x20/0x20 [1488105.012199] [] ? wake_up_state+0x20/0x20 [1488105.012238] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.012242] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.012245] [] ? wake_up_state+0x20/0x20 [1488105.012285] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.012324] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.012362] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.012366] [] ? insert_kthread_work+0x40/0x40 [1488105.012405] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.012408] [] ? kthread+0xd1/0xe0 [1488105.012410] [] ? kthread+0xd1/0xe0 [1488105.012451] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.012453] Task dump for CPU 13: [1488105.012456] [] ? kthread+0xd1/0xe0 [1488105.012459] [] ? insert_kthread_work+0x40/0x40 [1488105.012461] [] ? insert_kthread_work+0x40/0x40 [1488105.012465] [] ? wake_up_state+0x20/0x20 [1488105.012471] [] ? insert_kthread_work+0x40/0x40 [1488105.012474] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.012477] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.012517] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.012522] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.012525] [] ? insert_kthread_work+0x40/0x40 [1488105.012527] [] ? insert_kthread_work+0x40/0x40 [1488105.012531] [] ? kthread+0xd1/0xe0 [1488105.012535] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.012538] [] ? insert_kthread_work+0x40/0x40 [1488105.012541] Task dump for CPU 88: [1488105.012542] Task dump for CPU 69: [1488105.012546] [] ? insert_kthread_work+0x40/0x40 [1488105.012548] Call Trace: [1488105.012550] Task dump for CPU 47: [1488105.012554] ptlrpcd_00_00 R [1488105.012558] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.012597] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.012599] ptlrpcd_06_06 R [1488105.012601] ptlrpcd_07_02 R [1488105.012602] running task [1488105.012606] [] ? insert_kthread_work+0x40/0x40 [1488105.012645] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.012646] running task [1488105.012649] 0 54156 2 0x00000088 [1488105.012651] 0 54234 2 0x00000088 [1488105.012654] Task dump for CPU 26: [1488105.012692] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.012695] running task 0 54242 2 0x00000088 [1488105.012696] Call Trace: [1488105.012697] Call Trace: [1488105.012740] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.012741] Call Trace: [1488105.012782] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.012820] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.012826] [] ? wake_up_state+0x20/0x20 [1488105.012864] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.012903] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.012941] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.012944] ptlrpcd_01_00 R running task 0 54168 2 0x00000088 [1488105.012984] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.013022] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.013063] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.013100] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.013101] Call Trace: [1488105.013105] [] ? kthread+0xd1/0xe0 [1488105.013144] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.013185] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.013223] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.013262] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.013266] [] ? insert_kthread_work+0x40/0x40 [1488105.013305] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.013309] [] ? wake_up_state+0x20/0x20 [1488105.013311] [] ? wake_up_state+0x20/0x20 [1488105.013350] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.013354] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.013357] [] ? wake_up_state+0x20/0x20 [1488105.013398] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.013436] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.013475] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.013479] [] ? insert_kthread_work+0x40/0x40 [1488105.013518] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.013521] [] ? kthread+0xd1/0xe0 [1488105.013522] [] ? kthread+0xd1/0xe0 [1488105.013563] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.013565] Task dump for CPU 0: [1488105.013569] [] ? kthread+0xd1/0xe0 [1488105.013572] [] ? insert_kthread_work+0x40/0x40 [1488105.013574] [] ? insert_kthread_work+0x40/0x40 [1488105.013578] [] ? wake_up_state+0x20/0x20 [1488105.013583] [] ? insert_kthread_work+0x40/0x40 [1488105.013587] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.013589] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.013629] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.013634] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.013637] [] ? insert_kthread_work+0x40/0x40 [1488105.013639] [] ? insert_kthread_work+0x40/0x40 [1488105.013642] [] ? kthread+0xd1/0xe0 [1488105.013646] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.013649] [] ? insert_kthread_work+0x40/0x40 [1488105.013651] Task dump for CPU 95: [1488105.013652] Task dump for CPU 72: [1488105.013656] [] ? insert_kthread_work+0x40/0x40 [1488105.013659] Call Trace: [1488105.013661] Task dump for CPU 48: [1488105.013665] ptlrpcd_07_04 R [1488105.013669] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.013710] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.013712] ptlrpcd_00_04 R [1488105.013714] ptlrpcd_00_05 R [1488105.013715] running task [1488105.013719] [] ? insert_kthread_work+0x40/0x40 [1488105.013759] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.013760] running task [1488105.013763] 0 54244 2 0x00000088 [1488105.013764] 0 54160 2 0x00000088 [1488105.013767] Task dump for CPU 32: [1488105.013770] Task dump for CPU 16: [1488105.013808] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.013810] running task 0 54161 2 0x00000088 [1488105.013812] Call Trace: [1488105.013813] Call Trace: [1488105.013819] ptlrpcd_00_10 R [1488105.013858] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.013859] Call Trace: [1488105.013901] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.013939] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.013941] ptlrpcd_00_07 R [1488105.013944] running task [1488105.013947] [] ? wake_up_state+0x20/0x20 [1488105.013984] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.014023] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.014061] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.014064] 0 54166 2 0x00000088 [1488105.014068] running task 0 54163 2 0x00000088 [1488105.014107] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.014145] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.014187] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.014224] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.014226] Call Trace: [1488105.014228] Call Trace: [1488105.014231] [] ? kthread+0xd1/0xe0 [1488105.014269] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.014310] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.014348] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.014387] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.014425] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.014427] [] ? insert_kthread_work+0x40/0x40 [1488105.014467] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.014470] [] ? wake_up_state+0x20/0x20 [1488105.014472] [] ? wake_up_state+0x20/0x20 [1488105.014511] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.014549] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.014551] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.014555] [] ? wake_up_state+0x20/0x20 [1488105.014595] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.014633] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.014672] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.014710] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.014712] [] ? insert_kthread_work+0x40/0x40 [1488105.014751] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.014754] [] ? kthread+0xd1/0xe0 [1488105.014756] [] ? kthread+0xd1/0xe0 [1488105.014796] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.014836] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.014837] Task dump for CPU 5: [1488105.014840] [] ? kthread+0xd1/0xe0 [1488105.014843] [] ? insert_kthread_work+0x40/0x40 [1488105.014845] [] ? insert_kthread_work+0x40/0x40 [1488105.014849] [] ? wake_up_state+0x20/0x20 [1488105.014853] [] ? wake_up_state+0x20/0x20 [1488105.014856] [] ? insert_kthread_work+0x40/0x40 [1488105.014860] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.014862] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.014902] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.014941] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.014945] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.014948] [] ? insert_kthread_work+0x40/0x40 [1488105.014951] [] ? insert_kthread_work+0x40/0x40 [1488105.014954] [] ? kthread+0xd1/0xe0 [1488105.014958] [] ? kthread+0xd1/0xe0 [1488105.014960] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.014963] [] ? insert_kthread_work+0x40/0x40 [1488105.014965] Task dump for CPU 79: [1488105.014969] [] ? insert_kthread_work+0x40/0x40 [1488105.014973] [] ? insert_kthread_work+0x40/0x40 [1488105.014973] Call Trace: [1488105.014976] Task dump for CPU 56: [1488105.014982] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.014986] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.015026] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.015027] ptlrpcd_07_00 R [1488105.015030] ptlrpcd_00_03 R [1488105.015034] [] ? insert_kthread_work+0x40/0x40 [1488105.015038] [] ? insert_kthread_work+0x40/0x40 [1488105.015079] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.015080] running task [1488105.015083] 0 54240 2 0x00000088 [1488105.015086] Task dump for CPU 40: [1488105.015089] Task dump for CPU 19: [1488105.015126] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.015129] running task 0 54159 2 0x00000088 [1488105.015130] Call Trace: [1488105.015136] ptlrpcd_00_09 R [1488105.015176] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.015177] Call Trace: [1488105.015216] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.015218] ptlrpcd_05_02 R [1488105.015220] running task [1488105.015223] [] ? wake_up_state+0x20/0x20 [1488105.015261] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.015299] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.015303] 0 54165 2 0x00000088 [1488105.015307] running task 0 54218 2 0x00000088 [1488105.015346] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.015384] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.015421] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.015423] Call Trace: [1488105.015425] Call Trace: [1488105.015428] [] ? kthread+0xd1/0xe0 [1488105.015466] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.015505] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.015545] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.015583] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.015585] [] ? insert_kthread_work+0x40/0x40 [1488105.015624] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.015627] [] ? wake_up_state+0x20/0x20 [1488105.015667] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.015670] [] ? del_timer_sync+0x52/0x60 [1488105.015672] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.015676] [] ? wake_up_state+0x20/0x20 [1488105.015715] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.015754] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.015792] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.015795] [] ? insert_kthread_work+0x40/0x40 [1488105.015834] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.015836] [] ? kthread+0xd1/0xe0 [1488105.015877] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.015915] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.015916] Task dump for CPU 8: [1488105.015918] [] ? kthread+0xd1/0xe0 [1488105.015921] [] ? insert_kthread_work+0x40/0x40 [1488105.015925] [] ? wake_up_state+0x20/0x20 [1488105.015965] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.015969] [] ? insert_kthread_work+0x40/0x40 [1488105.015972] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.016013] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.016016] [] ? wake_up_state+0x20/0x20 [1488105.016020] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.016023] [] ? insert_kthread_work+0x40/0x40 [1488105.016027] [] ? kthread+0xd1/0xe0 [1488105.016067] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.016069] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.016073] [] ? insert_kthread_work+0x40/0x40 [1488105.016075] Task dump for CPU 80: [1488105.016079] [] ? insert_kthread_work+0x40/0x40 [1488105.016082] [] ? kthread+0xd1/0xe0 [1488105.016083] Call Trace: [1488105.016085] Task dump for CPU 61: [1488105.016091] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.016095] [] ? insert_kthread_work+0x40/0x40 [1488105.016133] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.016135] ptlrpcd_00_01 R [1488105.016138] ptlrpcd_06_08 R [1488105.016141] [] ? insert_kthread_work+0x40/0x40 [1488105.016145] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.016183] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.016184] running task [1488105.016188] 0 54157 2 0x00000088 [1488105.016190] Task dump for CPU 42: [1488105.016194] [] ? insert_kthread_work+0x40/0x40 [1488105.016232] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.016235] running task 0 54236 2 0x00000088 [1488105.016236] Call Trace: [1488105.016242] Task dump for CPU 21: [1488105.016281] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.016282] Call Trace: [1488105.016321] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.016326] ptlrpcd_01_01 R running task [1488105.016328] [] ? wake_up_state+0x20/0x20 [1488105.016366] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.016405] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.016407] 0 54169 2 0x00000088 [1488105.016449] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.016452] [] ? del_timer_sync+0x52/0x60 [1488105.016489] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.016491] Call Trace: [1488105.016495] ptlrpcd_06_11 R running task 0 54239 2 0x00000088 [1488105.016497] [] ? kthread+0xd1/0xe0 [1488105.016535] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.016574] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.016614] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.016615] Call Trace: [1488105.016618] [] ? insert_kthread_work+0x40/0x40 [1488105.016656] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.016659] [] ? wake_up_state+0x20/0x20 [1488105.016699] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.016737] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.016739] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.016779] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.016818] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.016857] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.016895] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.016897] [] ? insert_kthread_work+0x40/0x40 [1488105.016901] [] ? wake_up_state+0x20/0x20 [1488105.016903] [] ? kthread+0xd1/0xe0 [1488105.016945] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.016983] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.016984] Task dump for CPU 13: [1488105.017023] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.017026] [] ? insert_kthread_work+0x40/0x40 [1488105.017030] [] ? wake_up_state+0x20/0x20 [1488105.017070] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.017074] [] ? kthread+0xd1/0xe0 [1488105.017078] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.017118] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.017122] [] ? wake_up_state+0x20/0x20 [1488105.017125] [] ? insert_kthread_work+0x40/0x40 [1488105.017128] [] ? insert_kthread_work+0x40/0x40 [1488105.017131] [] ? kthread+0xd1/0xe0 [1488105.017172] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.017174] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.017178] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.017180] Task dump for CPU 88: [1488105.017184] [] ? insert_kthread_work+0x40/0x40 [1488105.017187] [] ? kthread+0xd1/0xe0 [1488105.017188] Call Trace: [1488105.017191] [] ? insert_kthread_work+0x40/0x40 [1488105.017197] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.017201] [] ? insert_kthread_work+0x40/0x40 [1488105.017241] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.017243] Task dump for CPU 64: [1488105.017249] [] ? insert_kthread_work+0x40/0x40 [1488105.017253] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.017290] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.017292] ptlrpcd_00_00 R running task [1488105.017296] 0 54156 2 0x00000088 [1488105.017298] Task dump for CPU 43: [1488105.017302] [] ? insert_kthread_work+0x40/0x40 [1488105.017340] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.017343] Call Trace: [1488105.017346] ptlrpcd_00_06 R running task [1488105.017349] Task dump for CPU 24: [1488105.017389] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.017391] 0 54162 2 0x00000088 [1488105.017429] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.017434] ptlrpcd_05_10 R running task [1488105.017437] [] ? wake_up_state+0x20/0x20 [1488105.017438] Call Trace: [1488105.017477] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.017479] 0 54226 2 0x00000088 [1488105.017521] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.017559] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.017597] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.017598] Call Trace: [1488105.017602] ptlrpcd_00_11 R running task 0 54167 2 0x00000088 [1488105.017604] [] ? kthread+0xd1/0xe0 [1488105.017643] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.017682] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.017686] Call Trace: [1488105.017689] [] ? insert_kthread_work+0x40/0x40 [1488105.017727] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.017730] [] ? wake_up_state+0x20/0x20 [1488105.017733] [] sched_show_task+0xa8/0x110 [1488105.017772] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.017775] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.017814] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.017853] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.017857] [] dump_cpu_task+0x39/0x70 [1488105.017861] [] ? del_timer_sync+0x52/0x60 [1488105.017863] [] ? insert_kthread_work+0x40/0x40 [1488105.017866] [] ? wake_up_state+0x20/0x20 [1488105.017869] [] ? kthread+0xd1/0xe0 [1488105.017872] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.017911] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.017912] Task dump for CPU 0: [1488105.017952] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.017954] [] ? insert_kthread_work+0x40/0x40 [1488105.017958] [] rcu_check_callbacks+0x442/0x730 [1488105.017996] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.018000] [] ? kthread+0xd1/0xe0 [1488105.018003] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.018008] [] ? tick_sched_do_timer+0x50/0x50 [1488105.018048] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.018051] [] ? insert_kthread_work+0x40/0x40 [1488105.018054] [] ? insert_kthread_work+0x40/0x40 [1488105.018058] [] update_process_times+0x46/0x80 [1488105.018062] [] ? wake_up_state+0x20/0x20 [1488105.018064] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.018067] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.018070] Task dump for CPU 95: [1488105.018073] [] tick_sched_handle+0x30/0x70 [1488105.018113] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.018113] Call Trace: [1488105.018116] [] ? insert_kthread_work+0x40/0x40 [1488105.018122] [] tick_sched_timer+0x39/0x80 [1488105.018125] [] ? kthread+0xd1/0xe0 [1488105.018163] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.018165] Task dump for CPU 69: [1488105.018171] [] __hrtimer_run_queues+0xf3/0x270 [1488105.018174] [] ? insert_kthread_work+0x40/0x40 [1488105.018211] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.018212] ptlrpcd_07_04 R running task [1488105.018215] 0 54244 2 0x00000088 [1488105.018219] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.018223] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.018260] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.018263] Call Trace: [1488105.018267] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.018271] [] ? insert_kthread_work+0x40/0x40 [1488105.018309] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.018311] ptlrpcd_06_06 R running task 0 54234 2 0x00000088 [1488105.018349] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.018352] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.018356] Task dump for CPU 26: [1488105.018358] [] ? wake_up_state+0x20/0x20 [1488105.018360] Call Trace: [1488105.018398] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.018401] [] apic_timer_interrupt+0x162/0x170 [1488105.018442] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.018481] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.018518] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.018521] ptlrpcd_01_00 R [1488105.018523] [1488105.018526] [] ? kthread+0xd1/0xe0 [1488105.018564] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.018603] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.018608] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [1488105.018611] running task 0 54168 2 0x00000088 [1488105.018613] [] ? insert_kthread_work+0x40/0x40 [1488105.018651] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.018654] [] ? wake_up_state+0x20/0x20 [1488105.018658] [] queued_spin_lock_slowpath+0xb/0xf [1488105.018660] Call Trace: [1488105.018663] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.018702] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.018741] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.018745] [] _raw_spin_lock+0x20/0x30 [1488105.018783] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.018785] [] ? insert_kthread_work+0x40/0x40 [1488105.018788] [] ? wake_up_state+0x20/0x20 [1488105.018791] [] ? kthread+0xd1/0xe0 [1488105.018800] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.018839] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.018840] Task dump for CPU 5: [1488105.018879] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.018882] [] ? insert_kthread_work+0x40/0x40 [1488105.018894] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.018933] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.018937] [] ? kthread+0xd1/0xe0 [1488105.018940] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.018979] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.019019] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.019024] [] ? insert_kthread_work+0x40/0x40 [1488105.019027] [] ? insert_kthread_work+0x40/0x40 [1488105.019066] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.019070] [] ? wake_up_state+0x20/0x20 [1488105.019071] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.019075] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.019114] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.019153] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.019154] Call Trace: [1488105.019157] [] ? insert_kthread_work+0x40/0x40 [1488105.019197] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.019200] [] ? kthread+0xd1/0xe0 [1488105.019238] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.019240] Task dump for CPU 72: [1488105.019245] [] ? wake_up_state+0x20/0x20 [1488105.019248] [] ? insert_kthread_work+0x40/0x40 [1488105.019285] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.019328] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.019332] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.019368] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.019373] [] kthread+0xd1/0xe0 [1488105.019377] [] ? insert_kthread_work+0x40/0x40 [1488105.019419] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.019422] ptlrpcd_00_04 R running task 0 54160 2 0x00000088 [1488105.019426] [] ? insert_kthread_work+0x40/0x40 [1488105.019428] Task dump for CPU 16: [1488105.019430] [] ? wake_up_state+0x20/0x20 [1488105.019432] Call Trace: [1488105.019436] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.019481] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.019519] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.019522] [] ? insert_kthread_work+0x40/0x40 [1488105.019527] [] ? kthread+0xd1/0xe0 [1488105.019565] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.019568] Task dump for CPU 47: [1488105.019571] ptlrpcd_00_07 R running task 0 54163 2 0x00000088 [1488105.019574] [] ? insert_kthread_work+0x40/0x40 [1488105.019612] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.019616] Call Trace: [1488105.019619] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.019658] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.019703] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.019705] [] ? insert_kthread_work+0x40/0x40 [1488105.019708] [] ? wake_up_state+0x20/0x20 [1488105.019711] ptlrpcd_07_02 R running task 0 54242 2 0x00000088 [1488105.019754] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.019756] Task dump for CPU 8: [1488105.019795] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.019797] Call Trace: [1488105.019839] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.019844] [] ? kthread+0xd1/0xe0 [1488105.019884] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.019927] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.019931] [] ? insert_kthread_work+0x40/0x40 [1488105.019971] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.019974] [] ? wake_up_state+0x20/0x20 [1488105.019976] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.019979] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.020018] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.020061] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.020062] Call Trace: [1488105.020066] [] ? insert_kthread_work+0x40/0x40 [1488105.020106] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.020109] [] ? kthread+0xd1/0xe0 [1488105.020148] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.020150] Task dump for CPU 79: [1488105.020155] [] ? wake_up_state+0x20/0x20 [1488105.020158] [] ? insert_kthread_work+0x40/0x40 [1488105.020200] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.020241] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.020245] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.020286] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.020290] [] ? kthread+0xd1/0xe0 [1488105.020294] [] ? insert_kthread_work+0x40/0x40 [1488105.020335] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.020338] ptlrpcd_07_00 R running task 0 54240 2 0x00000088 [1488105.020343] [] ? insert_kthread_work+0x40/0x40 [1488105.020345] Task dump for CPU 19: [1488105.020348] [] ? wake_up_state+0x20/0x20 [1488105.020349] Call Trace: [1488105.020353] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.020396] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.020434] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.020438] [] ? insert_kthread_work+0x40/0x40 [1488105.020443] [] ? kthread+0xd1/0xe0 [1488105.020481] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.020483] Task dump for CPU 48: [1488105.020486] Task dump for CPU 32: [1488105.020488] ptlrpcd_05_02 R running task 0 54218 2 0x00000088 [1488105.020490] [] ? insert_kthread_work+0x40/0x40 [1488105.020528] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.020533] ptlrpcd_00_05 R [1488105.020534] Call Trace: [1488105.020537] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.020576] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.020578] ptlrpcd_00_10 R [1488105.020581] running task [1488105.020620] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.020621] [] ? insert_kthread_work+0x40/0x40 [1488105.020625] [] ? wake_up_state+0x20/0x20 [1488105.020627] 0 54161 2 0x00000088 [1488105.020631] running task 0 54166 2 0x00000088 [1488105.020633] [] ? del_timer_sync+0x52/0x60 [1488105.020634] Task dump for CPU 13: [1488105.020673] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.020675] Call Trace: [1488105.020678] Call Trace: [1488105.020719] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.020723] [] ? kthread+0xd1/0xe0 [1488105.020763] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.020801] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.020842] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.020846] [] ? insert_kthread_work+0x40/0x40 [1488105.020886] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.020924] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.020965] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.020967] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.020971] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.021010] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.021048] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.021049] [] ? wake_up_state+0x20/0x20 [1488105.021050] Call Trace: [1488105.021053] [] ? insert_kthread_work+0x40/0x40 [1488105.021094] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.021134] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.021175] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.021216] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.021218] Task dump for CPU 80: [1488105.021222] [] ? wake_up_state+0x20/0x20 [1488105.021226] [] ? wake_up_state+0x20/0x20 [1488105.021228] [] ? kthread+0xd1/0xe0 [1488105.021265] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.021307] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.021347] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.021348] [] ? insert_kthread_work+0x40/0x40 [1488105.021389] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.021393] [] ? kthread+0xd1/0xe0 [1488105.021396] [] ? kthread+0xd1/0xe0 [1488105.021399] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.021441] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.021443] ptlrpcd_00_01 R running task 0 54157 2 0x00000088 [1488105.021447] [] ? insert_kthread_work+0x40/0x40 [1488105.021451] [] ? insert_kthread_work+0x40/0x40 [1488105.021453] [] ? insert_kthread_work+0x40/0x40 [1488105.021455] [] ? wake_up_state+0x20/0x20 [1488105.021457] Call Trace: [1488105.021462] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.021466] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.021467] Task dump for CPU 21: [1488105.021508] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.021546] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.021549] [] ? insert_kthread_work+0x40/0x40 [1488105.021553] [] ? insert_kthread_work+0x40/0x40 [1488105.021557] [] ? kthread+0xd1/0xe0 [1488105.021595] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.021597] Task dump for CPU 56: [1488105.021600] Task dump for CPU 40: [1488105.021604] [] ? insert_kthread_work+0x40/0x40 [1488105.021642] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.021644] ptlrpcd_06_11 R running task [1488105.021648] ptlrpcd_00_03 R [1488105.021649] 0 54239 2 0x00000088 [1488105.021652] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.021691] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.021692] ptlrpcd_00_09 R [1488105.021695] running task [1488105.021696] Call Trace: [1488105.021698] [] ? insert_kthread_work+0x40/0x40 [1488105.021701] [] ? wake_up_state+0x20/0x20 [1488105.021704] 0 54159 2 0x00000088 [1488105.021708] running task 0 54165 2 0x00000088 [1488105.021749] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.021750] Task dump for CPU 0: [1488105.021789] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.021791] Call Trace: [1488105.021793] Call Trace: [1488105.021832] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.021835] [] ? kthread+0xd1/0xe0 [1488105.021875] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.021913] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.021951] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.021954] [] ? insert_kthread_work+0x40/0x40 [1488105.021993] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.021996] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.022033] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.022071] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.022075] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.022113] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.022115] Call Trace: [1488105.022152] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.022154] [] ? wake_up_state+0x20/0x20 [1488105.022157] [] ? insert_kthread_work+0x40/0x40 [1488105.022197] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.022236] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.022274] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.022312] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.022314] Task dump for CPU 88: [1488105.022318] [] ? wake_up_state+0x20/0x20 [1488105.022358] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.022359] [] ? wake_up_state+0x20/0x20 [1488105.022362] [] ? kthread+0xd1/0xe0 [1488105.022404] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.022442] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.022480] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.022482] [] ? insert_kthread_work+0x40/0x40 [1488105.022486] [] ? kthread+0xd1/0xe0 [1488105.022526] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.022528] [] ? kthread+0xd1/0xe0 [1488105.022530] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.022533] ptlrpcd_00_00 R running task 0 54156 2 0x00000088 [1488105.022537] [] ? insert_kthread_work+0x40/0x40 [1488105.022541] [] ? wake_up_state+0x20/0x20 [1488105.022543] [] ? insert_kthread_work+0x40/0x40 [1488105.022545] [] ? insert_kthread_work+0x40/0x40 [1488105.022546] Call Trace: [1488105.022551] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.022590] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.022593] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.022593] Task dump for CPU 24: [1488105.022632] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.022635] [] ? insert_kthread_work+0x40/0x40 [1488105.022638] [] ? kthread+0xd1/0xe0 [1488105.022640] [] ? insert_kthread_work+0x40/0x40 [1488105.022680] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.022682] Task dump for CPU 61: [1488105.022687] [] ? insert_kthread_work+0x40/0x40 [1488105.022688] Task dump for CPU 42: [1488105.022726] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.022730] ptlrpcd_00_11 R running task [1488105.022734] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.022736] ptlrpcd_06_08 R [1488105.022737] 0 54167 2 0x00000088 [1488105.022776] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.022778] ptlrpcd_01_01 R [1488105.022783] [] ? insert_kthread_work+0x40/0x40 [1488105.022783] running task [1488105.022784] Call Trace: [1488105.022787] [] ? wake_up_state+0x20/0x20 [1488105.022790] 0 54236 2 0x00000088 [1488105.022792] Task dump for CPU 5: [1488105.022794] running task 0 54169 2 0x00000088 [1488105.022831] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.022870] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.022872] Call Trace: [1488105.022876] Call Trace: [1488105.022878] [] ? del_timer_sync+0x52/0x60 [1488105.022881] [] ? kthread+0xd1/0xe0 [1488105.022920] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.022960] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.022996] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.022999] [] ? insert_kthread_work+0x40/0x40 [1488105.023002] [] ? del_timer_sync+0x52/0x60 [1488105.023006] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.023043] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.023080] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.023083] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.023123] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.023125] Call Trace: [1488105.023161] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.023199] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.023202] [] ? insert_kthread_work+0x40/0x40 [1488105.023241] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.023279] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.023317] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.023319] [] ? wake_up_state+0x20/0x20 [1488105.023321] Task dump for CPU 95: [1488105.023361] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.023400] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.023402] [] ? wake_up_state+0x20/0x20 [1488105.023440] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.023446] [] ? wake_up_state+0x20/0x20 [1488105.023485] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.023523] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.023525] [] ? kthread+0xd1/0xe0 [1488105.023567] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.023606] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.023608] [] ? kthread+0xd1/0xe0 [1488105.023610] [] ? insert_kthread_work+0x40/0x40 [1488105.023612] ptlrpcd_07_04 R running task 0 54244 2 0x00000088 [1488105.023617] [] ? kthread+0xd1/0xe0 [1488105.023620] [] ? wake_up_state+0x20/0x20 [1488105.023622] [] ? insert_kthread_work+0x40/0x40 [1488105.023625] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.023627] Call Trace: [1488105.023631] [] ? insert_kthread_work+0x40/0x40 [1488105.023670] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.023672] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.023675] [] ? insert_kthread_work+0x40/0x40 [1488105.023714] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.023717] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.023721] [] ? kthread+0xd1/0xe0 [1488105.023723] [] ? insert_kthread_work+0x40/0x40 [1488105.023723] Task dump for CPU 26: [1488105.023761] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.023764] [] ? insert_kthread_work+0x40/0x40 [1488105.023768] [] ? insert_kthread_work+0x40/0x40 [1488105.023769] Task dump for CPU 43: [1488105.023808] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.023810] Task dump for CPU 64: [1488105.023814] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.023816] ptlrpcd_01_00 R [1488105.023818] ptlrpcd_05_10 R [1488105.023858] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.023860] running task [1488105.023864] [] ? insert_kthread_work+0x40/0x40 [1488105.023865] ptlrpcd_00_06 R [1488105.023867] 0 54168 2 0x00000088 [1488105.023870] [] ? wake_up_state+0x20/0x20 [1488105.023873] running task [1488105.023876] Task dump for CPU 8: [1488105.023877] 0 54226 2 0x00000088 [1488105.023878] Call Trace: [1488105.023917] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.023920] running task 0 54162 2 0x00000088 [1488105.023924] Call Trace: [1488105.023961] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.023963] [] ? kthread+0xd1/0xe0 [1488105.023965] Call Trace: [1488105.024005] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.024042] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.024045] [] ? insert_kthread_work+0x40/0x40 [1488105.024084] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.024087] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.024125] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.024161] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.024165] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.024203] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.024205] Call Trace: [1488105.024242] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.024280] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.024283] [] ? insert_kthread_work+0x40/0x40 [1488105.024322] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.024361] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.024399] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.024402] [] ? wake_up_state+0x20/0x20 [1488105.024442] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.024481] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.024483] [] ? wake_up_state+0x20/0x20 [1488105.024522] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.024526] [] ? wake_up_state+0x20/0x20 [1488105.024565] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.024603] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.024605] [] ? kthread+0xd1/0xe0 [1488105.024645] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.024685] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.024686] [] ? kthread+0xd1/0xe0 [1488105.024689] [] ? insert_kthread_work+0x40/0x40 [1488105.024693] [] ? kthread+0xd1/0xe0 [1488105.024697] [] ? wake_up_state+0x20/0x20 [1488105.024699] [] ? insert_kthread_work+0x40/0x40 [1488105.024702] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.024706] [] ? insert_kthread_work+0x40/0x40 [1488105.024745] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.024747] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.024750] [] ? insert_kthread_work+0x40/0x40 [1488105.024753] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.024757] [] ? kthread+0xd1/0xe0 [1488105.024759] [] ? insert_kthread_work+0x40/0x40 [1488105.024760] Task dump for CPU 16: [1488105.024763] [] ? insert_kthread_work+0x40/0x40 [1488105.024767] [] ? insert_kthread_work+0x40/0x40 [1488105.024768] Task dump for CPU 47: [1488105.024772] Task dump for CPU 69: [1488105.024776] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.024778] ptlrpcd_00_07 R [1488105.024780] ptlrpcd_07_02 R [1488105.024782] running task [1488105.024786] [] ? insert_kthread_work+0x40/0x40 [1488105.024787] ptlrpcd_06_06 R [1488105.024789] 0 54163 2 0x00000088 [1488105.024791] running task [1488105.024795] Task dump for CPU 13: [1488105.024796] 0 54242 2 0x00000088 [1488105.024797] Call Trace: [1488105.024801] running task 0 54234 2 0x00000088 [1488105.024805] Call Trace: [1488105.024844] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.024846] Call Trace: [1488105.024886] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.024923] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.024962] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.024965] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.025002] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.025040] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.025079] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.025080] Call Trace: [1488105.025117] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.025156] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.025195] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.025235] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.025273] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.025276] [] ? wake_up_state+0x20/0x20 [1488105.025316] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.025355] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.025357] [] ? wake_up_state+0x20/0x20 [1488105.025396] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.025400] [] ? wake_up_state+0x20/0x20 [1488105.025440] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.025478] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.025480] [] ? kthread+0xd1/0xe0 [1488105.025521] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.025562] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.025564] [] ? kthread+0xd1/0xe0 [1488105.025566] [] ? insert_kthread_work+0x40/0x40 [1488105.025570] [] ? kthread+0xd1/0xe0 [1488105.025573] [] ? wake_up_state+0x20/0x20 [1488105.025575] [] ? insert_kthread_work+0x40/0x40 [1488105.025578] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.025582] [] ? insert_kthread_work+0x40/0x40 [1488105.025623] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.025625] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.025627] [] ? insert_kthread_work+0x40/0x40 [1488105.025631] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.025634] [] ? kthread+0xd1/0xe0 [1488105.025636] [] ? insert_kthread_work+0x40/0x40 [1488105.025637] Task dump for CPU 19: [1488105.025640] [] ? insert_kthread_work+0x40/0x40 [1488105.025644] [] ? insert_kthread_work+0x40/0x40 [1488105.025644] Task dump for CPU 48: [1488105.025647] Task dump for CPU 32: [1488105.025651] Task dump for CPU 72: [1488105.025655] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.025656] ptlrpcd_05_02 R [1488105.025658] ptlrpcd_00_05 R [1488105.025659] ptlrpcd_00_10 R [1488105.025662] running task [1488105.025666] [] ? insert_kthread_work+0x40/0x40 [1488105.025666] ptlrpcd_00_04 R [1488105.025668] running task [1488105.025670] 0 54218 2 0x00000088 [1488105.025673] running task [1488105.025676] Task dump for CPU 0: [1488105.025679] 0 54161 2 0x00000088 [1488105.025681] 0 54166 2 0x00000088 [1488105.025682] Call Trace: [1488105.025685] running task 0 54160 2 0x00000088 [1488105.025690] Call Trace: [1488105.025690] Call Trace: [1488105.025730] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.025731] Call Trace: [1488105.025772] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.025809] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.025811] [] ? del_timer_sync+0x52/0x60 [1488105.025850] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.025854] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.025893] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.025930] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.025967] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.026006] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.026007] Call Trace: [1488105.026046] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.026083] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.026121] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.026159] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.026202] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.026241] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.026280] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.026319] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.026359] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.026402] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.026405] [] ? wake_up_state+0x20/0x20 [1488105.026407] [] ? wake_up_state+0x20/0x20 [1488105.026410] [] ? wake_up_state+0x20/0x20 [1488105.026414] [] ? wake_up_state+0x20/0x20 [1488105.026457] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.026496] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.026534] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.026573] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.026613] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.026657] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.026660] [] ? kthread+0xd1/0xe0 [1488105.026661] [] ? kthread+0xd1/0xe0 [1488105.026664] [] ? kthread+0xd1/0xe0 [1488105.026667] [] ? kthread+0xd1/0xe0 [1488105.026671] [] ? wake_up_state+0x20/0x20 [1488105.026674] [] ? insert_kthread_work+0x40/0x40 [1488105.026676] [] ? insert_kthread_work+0x40/0x40 [1488105.026679] [] ? insert_kthread_work+0x40/0x40 [1488105.026682] [] ? insert_kthread_work+0x40/0x40 [1488105.026725] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.026729] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.026731] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.026733] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.026738] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.026741] [] ? kthread+0xd1/0xe0 [1488105.026744] [] ? insert_kthread_work+0x40/0x40 [1488105.026746] [] ? insert_kthread_work+0x40/0x40 [1488105.026749] [] ? insert_kthread_work+0x40/0x40 [1488105.026752] [] ? insert_kthread_work+0x40/0x40 [1488105.026756] [] ? insert_kthread_work+0x40/0x40 [1488105.026758] Task dump for CPU 56: [1488105.026759] Task dump for CPU 40: [1488105.026760] Task dump for CPU 21: [1488105.026763] Task dump for CPU 79: [1488105.026767] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.026771] ptlrpcd_00_03 R [1488105.026772] ptlrpcd_00_09 R [1488105.026774] ptlrpcd_06_11 R [1488105.026778] [] ? insert_kthread_work+0x40/0x40 [1488105.026780] ptlrpcd_07_00 R [1488105.026781] running task [1488105.026782] running task [1488105.026784] running task [1488105.026787] Task dump for CPU 5: [1488105.026791] 0 54159 2 0x00000088 [1488105.026792] 0 54165 2 0x00000088 [1488105.026794] 0 54239 2 0x00000088 [1488105.026798] running task 0 54240 2 0x00000088 [1488105.026802] Call Trace: [1488105.026804] Call Trace: [1488105.026804] Call Trace: [1488105.026806] Call Trace: [1488105.026847] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.026883] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.026921] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.026960] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.026963] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.027002] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.027039] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.027077] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.027115] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.027117] Call Trace: [1488105.027155] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.027192] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.027230] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.027268] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.027311] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.027350] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.027389] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.027429] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.027469] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.027512] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.027515] [] ? wake_up_state+0x20/0x20 [1488105.027517] [] ? wake_up_state+0x20/0x20 [1488105.027520] [] ? wake_up_state+0x20/0x20 [1488105.027523] [] ? wake_up_state+0x20/0x20 [1488105.027566] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.027604] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.027642] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.027682] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.027722] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.027765] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.027768] [] ? kthread+0xd1/0xe0 [1488105.027770] [] ? kthread+0xd1/0xe0 [1488105.027772] [] ? kthread+0xd1/0xe0 [1488105.027776] [] ? kthread+0xd1/0xe0 [1488105.027780] [] ? wake_up_state+0x20/0x20 [1488105.027783] [] ? insert_kthread_work+0x40/0x40 [1488105.027784] [] ? insert_kthread_work+0x40/0x40 [1488105.027786] [] ? insert_kthread_work+0x40/0x40 [1488105.027790] [] ? insert_kthread_work+0x40/0x40 [1488105.027834] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.027837] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.027839] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.027842] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.027846] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.027850] [] ? kthread+0xd1/0xe0 [1488105.027853] [] ? insert_kthread_work+0x40/0x40 [1488105.027854] [] ? insert_kthread_work+0x40/0x40 [1488105.027857] [] ? insert_kthread_work+0x40/0x40 [1488105.027861] [] ? insert_kthread_work+0x40/0x40 [1488105.027865] [] ? insert_kthread_work+0x40/0x40 [1488105.027867] Task dump for CPU 61: [1488105.027868] Task dump for CPU 42: [1488105.027869] Task dump for CPU 24: [1488105.027871] Task dump for CPU 80: [1488105.027875] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.027879] ptlrpcd_06_08 R [1488105.027881] ptlrpcd_01_01 R [1488105.027883] ptlrpcd_00_11 R [1488105.027887] [] ? insert_kthread_work+0x40/0x40 [1488105.027889] ptlrpcd_00_01 R [1488105.027891] running task [1488105.027892] running task [1488105.027894] running task [1488105.027896] Task dump for CPU 8: [1488105.027899] 0 54236 2 0x00000088 [1488105.027901] 0 54169 2 0x00000088 [1488105.027903] 0 54167 2 0x00000088 [1488105.027907] running task 0 54157 2 0x00000088 [1488105.027911] Call Trace: [1488105.027912] Call Trace: [1488105.027913] Call Trace: [1488105.027916] Call Trace: [1488105.027957] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.027993] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.028031] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.028069] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.028073] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.028076] [] ? del_timer_sync+0x52/0x60 [1488105.028113] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.028115] [] ? del_timer_sync+0x52/0x60 [1488105.028154] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.028156] Call Trace: [1488105.028196] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.028233] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.028271] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.028309] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.028352] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.028390] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.028428] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.028466] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.028506] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.028548] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.028587] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.028590] [] ? wake_up_state+0x20/0x20 [1488105.028629] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.028633] [] ? wake_up_state+0x20/0x20 [1488105.028676] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.028679] [] ? wake_up_state+0x20/0x20 [1488105.028717] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.028719] [] ? wake_up_state+0x20/0x20 [1488105.028760] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.028803] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.028842] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.028844] [] ? kthread+0xd1/0xe0 [1488105.028883] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.028887] [] ? kthread+0xd1/0xe0 [1488105.028891] [] ? wake_up_state+0x20/0x20 [1488105.028894] [] ? kthread+0xd1/0xe0 [1488105.028896] [] ? insert_kthread_work+0x40/0x40 [1488105.028898] [] ? kthread+0xd1/0xe0 [1488105.028902] [] ? insert_kthread_work+0x40/0x40 [1488105.028945] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.028949] [] ? insert_kthread_work+0x40/0x40 [1488105.028951] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.028953] [] ? insert_kthread_work+0x40/0x40 [1488105.028957] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.028961] [] ? kthread+0xd1/0xe0 [1488105.028965] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.028966] [] ? insert_kthread_work+0x40/0x40 [1488105.028969] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.028973] [] ? insert_kthread_work+0x40/0x40 [1488105.028977] [] ? insert_kthread_work+0x40/0x40 [1488105.028980] [] ? insert_kthread_work+0x40/0x40 [1488105.028981] Task dump for CPU 43: [1488105.028983] [] ? insert_kthread_work+0x40/0x40 [1488105.028986] Task dump for CPU 88: [1488105.028990] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.028992] Task dump for CPU 64: [1488105.028994] Task dump for CPU 26: [1488105.028996] ptlrpcd_05_10 R [1488105.029000] [] ? insert_kthread_work+0x40/0x40 [1488105.029002] ptlrpcd_00_00 R [1488105.029003] ptlrpcd_00_06 R [1488105.029004] running task [1488105.029007] ptlrpcd_01_00 R [1488105.029010] Task dump for CPU 13: [1488105.029012] running task [1488105.029014] 0 54226 2 0x00000088 [1488105.029015] running task [1488105.029019] 0 54156 2 0x00000088 [1488105.029022] running task [1488105.029025] 0 54162 2 0x00000088 [1488105.029025] Call Trace: [1488105.029027] 0 54168 2 0x00000088 [1488105.029029] Call Trace: [1488105.029034] Call Trace: [1488105.029071] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.029071] Call Trace: [1488105.029111] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.029114] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.029153] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.029189] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.029228] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.029266] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.029269] Call Trace: [1488105.029307] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.029344] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.029382] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.029420] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.029460] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.029498] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.029536] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.029576] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.029580] [] sched_show_task+0xa8/0x110 [1488105.029620] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.029621] [] ? wake_up_state+0x20/0x20 [1488105.029661] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.029664] [] ? wake_up_state+0x20/0x20 [1488105.029668] [] dump_cpu_task+0x39/0x70 [1488105.029672] [] ? wake_up_state+0x20/0x20 [1488105.029709] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.029711] [] ? wake_up_state+0x20/0x20 [1488105.029751] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.029755] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.029794] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.029796] [] ? kthread+0xd1/0xe0 [1488105.029835] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.029838] [] ? kthread+0xd1/0xe0 [1488105.029842] [] rcu_check_callbacks+0x442/0x730 [1488105.029845] [] ? kthread+0xd1/0xe0 [1488105.029848] [] ? insert_kthread_work+0x40/0x40 [1488105.029849] [] ? kthread+0xd1/0xe0 [1488105.029853] [] ? insert_kthread_work+0x40/0x40 [1488105.029857] [] ? tick_sched_do_timer+0x50/0x50 [1488105.029860] [] ? insert_kthread_work+0x40/0x40 [1488105.029862] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.029865] [] ? insert_kthread_work+0x40/0x40 [1488105.029869] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.029872] [] update_process_times+0x46/0x80 [1488105.029875] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.029877] [] ? insert_kthread_work+0x40/0x40 [1488105.029880] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.029883] [] ? insert_kthread_work+0x40/0x40 [1488105.029886] [] tick_sched_handle+0x30/0x70 [1488105.029890] [] ? insert_kthread_work+0x40/0x40 [1488105.029891] Task dump for CPU 47: [1488105.029893] [] ? insert_kthread_work+0x40/0x40 [1488105.029895] Task dump for CPU 95: [1488105.029899] [] tick_sched_timer+0x39/0x80 [1488105.029902] Task dump for CPU 69: [1488105.029904] Task dump for CPU 16: [1488105.029906] ptlrpcd_07_02 R [1488105.029911] [] __hrtimer_run_queues+0xf3/0x270 [1488105.029913] ptlrpcd_07_04 R [1488105.029914] ptlrpcd_06_06 R [1488105.029916] running task [1488105.029917] ptlrpcd_00_07 R [1488105.029921] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.029924] running task [1488105.029925] 0 54242 2 0x00000088 [1488105.029926] running task [1488105.029930] 0 54244 2 0x00000088 [1488105.029935] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.029937] 0 54234 2 0x00000088 [1488105.029938] Call Trace: [1488105.029941] running task 0 54163 2 0x00000088 [1488105.029942] Call Trace: [1488105.029946] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.029948] Call Trace: [1488105.029990] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.029990] Call Trace: [1488105.030030] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.030034] [] apic_timer_interrupt+0x162/0x170 [1488105.030072] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.030113] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.030152] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.030155] [1488105.030194] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.030234] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.030236] [] sched_show_task+0xa8/0x110 [1488105.030275] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.030279] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.030318] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.030359] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.030362] [] dump_cpu_task+0x39/0x70 [1488105.030402] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.030405] [] queued_spin_lock_slowpath+0xb/0xf [1488105.030444] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.030446] [] ? wake_up_state+0x20/0x20 [1488105.030449] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.030452] [] ? wake_up_state+0x20/0x20 [1488105.030455] [] _raw_spin_lock+0x20/0x30 [1488105.030459] [] ? wake_up_state+0x20/0x20 [1488105.030500] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.030502] [] rcu_check_callbacks+0x442/0x730 [1488105.030542] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.030552] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.030591] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.030593] [] ? kthread+0xd1/0xe0 [1488105.030596] [] ? tick_sched_do_timer+0x50/0x50 [1488105.030599] [] ? kthread+0xd1/0xe0 [1488105.030612] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.030615] [] ? kthread+0xd1/0xe0 [1488105.030617] [] ? insert_kthread_work+0x40/0x40 [1488105.030620] [] update_process_times+0x46/0x80 [1488105.030623] [] ? insert_kthread_work+0x40/0x40 [1488105.030665] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.030669] [] ? insert_kthread_work+0x40/0x40 [1488105.030672] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.030673] [] tick_sched_handle+0x30/0x70 [1488105.030678] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.030721] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.030724] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.030727] [] ? insert_kthread_work+0x40/0x40 [1488105.030729] [] tick_sched_timer+0x39/0x80 [1488105.030732] [] ? insert_kthread_work+0x40/0x40 [1488105.030775] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.030778] [] ? insert_kthread_work+0x40/0x40 [1488105.030779] Task dump for CPU 48: [1488105.030780] Task dump for CPU 32: [1488105.030783] [] __hrtimer_run_queues+0xf3/0x270 [1488105.030826] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.030829] Task dump for CPU 72: [1488105.030830] ptlrpcd_00_05 R [1488105.030833] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.030836] [] ? wake_up_state+0x20/0x20 [1488105.030838] ptlrpcd_00_10 R [1488105.030839] ptlrpcd_00_04 R [1488105.030840] running task [1488105.030843] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.030887] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.030889] running task [1488105.030891] 0 54161 2 0x00000088 [1488105.030892] 0 54166 2 0x00000088 [1488105.030894] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.030897] [] kthread+0xd1/0xe0 [1488105.030900] running task 0 54160 2 0x00000088 [1488105.030901] Call Trace: [1488105.030902] Call Trace: [1488105.030905] [] apic_timer_interrupt+0x162/0x170 [1488105.030908] [] ? insert_kthread_work+0x40/0x40 [1488105.030910] Call Trace: [1488105.030951] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.030991] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.030996] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.031034] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.031074] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.031112] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.031115] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.031119] [] ? insert_kthread_work+0x40/0x40 [1488105.031157] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.031198] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.031235] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.031237] [] queued_spin_lock_slowpath+0xb/0xf [1488105.031239] Task dump for CPU 0: [1488105.031278] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.031319] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.031359] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.031361] [] _raw_spin_lock+0x20/0x30 [1488105.031404] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.031405] [] ? wake_up_state+0x20/0x20 [1488105.031407] [] ? wake_up_state+0x20/0x20 [1488105.031416] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.031420] [] ? wake_up_state+0x20/0x20 [1488105.031462] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.031501] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.031512] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.031515] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.031555] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.031556] [] ? kthread+0xd1/0xe0 [1488105.031558] [] ? kthread+0xd1/0xe0 [1488105.031596] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.031597] Call Trace: [1488105.031601] [] ? kthread+0xd1/0xe0 [1488105.031603] [] ? insert_kthread_work+0x40/0x40 [1488105.031605] [] ? insert_kthread_work+0x40/0x40 [1488105.031641] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.031680] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.031683] [] ? insert_kthread_work+0x40/0x40 [1488105.031686] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.031688] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.031725] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.031763] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.031767] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.031768] [] ? insert_kthread_work+0x40/0x40 [1488105.031770] [] ? insert_kthread_work+0x40/0x40 [1488105.031809] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.031847] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.031850] [] ? insert_kthread_work+0x40/0x40 [1488105.031851] Task dump for CPU 56: [1488105.031852] Task dump for CPU 40: [1488105.031855] [] ? wake_up_state+0x20/0x20 [1488105.031894] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.031897] Task dump for CPU 79: [1488105.031899] ptlrpcd_00_03 R [1488105.031938] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.031941] [] ? wake_up_state+0x20/0x20 [1488105.031943] ptlrpcd_00_09 R [1488105.031944] ptlrpcd_07_00 R [1488105.031945] running task [1488105.031947] [] kthread+0xd1/0xe0 [1488105.031987] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.031989] running task [1488105.031991] 0 54159 2 0x00000088 [1488105.031992] 0 54165 2 0x00000088 [1488105.031994] [] ? insert_kthread_work+0x40/0x40 [1488105.031997] [] ? kthread+0xd1/0xe0 [1488105.032001] running task 0 54240 2 0x00000088 [1488105.032001] Call Trace: [1488105.032002] Call Trace: [1488105.032004] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.032007] [] ? insert_kthread_work+0x40/0x40 [1488105.032009] Call Trace: [1488105.032051] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.032088] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.032090] [] ? insert_kthread_work+0x40/0x40 [1488105.032094] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.032132] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.032173] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.032210] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.032211] Task dump for CPU 19: [1488105.032215] [] ? insert_kthread_work+0x40/0x40 [1488105.032253] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.032294] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.032331] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.032334] Task dump for CPU 5: [1488105.032373] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.032414] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.032453] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.032456] ptlrpcd_05_02 R running task [1488105.032496] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.032497] [] ? wake_up_state+0x20/0x20 [1488105.032499] [] ? wake_up_state+0x20/0x20 [1488105.032501] 0 54218 2 0x00000088 [1488105.032506] [] ? wake_up_state+0x20/0x20 [1488105.032547] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.032586] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.032587] Call Trace: [1488105.032589] ptlrpcd_06_01 R running task 0 54229 2 0x00000088 [1488105.032629] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.032631] [] ? kthread+0xd1/0xe0 [1488105.032633] [] ? kthread+0xd1/0xe0 [1488105.032671] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.032672] Call Trace: [1488105.032676] [] ? kthread+0xd1/0xe0 [1488105.032678] [] ? insert_kthread_work+0x40/0x40 [1488105.032680] [] ? insert_kthread_work+0x40/0x40 [1488105.032682] [] ? del_timer_sync+0x52/0x60 [1488105.032721] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.032724] [] ? insert_kthread_work+0x40/0x40 [1488105.032727] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.032730] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.032767] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.032805] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.032808] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.032810] [] ? insert_kthread_work+0x40/0x40 [1488105.032812] [] ? insert_kthread_work+0x40/0x40 [1488105.032849] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.032887] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.032891] [] ? insert_kthread_work+0x40/0x40 [1488105.032891] Task dump for CPU 61: [1488105.032892] Task dump for CPU 42: [1488105.032931] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.032970] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.032972] Task dump for CPU 80: [1488105.032974] ptlrpcd_06_08 R [1488105.032976] [] ? wake_up_state+0x20/0x20 [1488105.032980] [] ? wake_up_state+0x20/0x20 [1488105.032982] ptlrpcd_01_01 R [1488105.032983] ptlrpcd_00_01 R [1488105.032984] running task [1488105.033022] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.033062] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.033064] running task [1488105.033066] 0 54236 2 0x00000088 [1488105.033067] 0 54169 2 0x00000088 [1488105.033069] [] ? kthread+0xd1/0xe0 [1488105.033072] [] ? kthread+0xd1/0xe0 [1488105.033075] running task 0 54157 2 0x00000088 [1488105.033076] Call Trace: [1488105.033076] Call Trace: [1488105.033079] [] ? insert_kthread_work+0x40/0x40 [1488105.033082] [] ? insert_kthread_work+0x40/0x40 [1488105.033084] Call Trace: [1488105.033125] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.033162] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.033164] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.033169] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.033173] [] ? del_timer_sync+0x52/0x60 [1488105.033211] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.033213] [] ? insert_kthread_work+0x40/0x40 [1488105.033216] [] ? insert_kthread_work+0x40/0x40 [1488105.033220] [] sched_show_task+0xa8/0x110 [1488105.033261] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.033298] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.033299] Task dump for CPU 21: [1488105.033301] Task dump for CPU 8: [1488105.033305] [] dump_cpu_task+0x39/0x70 [1488105.033345] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.033385] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.033388] ptlrpcd_06_11 R [1488105.033392] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.033433] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.033436] [] ? wake_up_state+0x20/0x20 [1488105.033437] ptlrpcd_00_02 R [1488105.033439] running task [1488105.033443] [] rcu_check_callbacks+0x442/0x730 [1488105.033445] [] ? wake_up_state+0x20/0x20 [1488105.033483] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.033485] 0 54239 2 0x00000088 [1488105.033489] running task 0 54158 2 0x00000088 [1488105.033492] [] ? tick_sched_do_timer+0x50/0x50 [1488105.033534] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.033535] [] ? kthread+0xd1/0xe0 [1488105.033536] Call Trace: [1488105.033538] Call Trace: [1488105.033542] [] update_process_times+0x46/0x80 [1488105.033543] [] ? kthread+0xd1/0xe0 [1488105.033545] [] ? insert_kthread_work+0x40/0x40 [1488105.033582] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.033620] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.033624] [] tick_sched_handle+0x30/0x70 [1488105.033627] [] ? insert_kthread_work+0x40/0x40 [1488105.033629] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.033666] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.033704] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.033707] [] tick_sched_timer+0x39/0x80 [1488105.033709] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.033711] [] ? insert_kthread_work+0x40/0x40 [1488105.033748] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.033786] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.033790] [] __hrtimer_run_queues+0xf3/0x270 [1488105.033792] [] ? insert_kthread_work+0x40/0x40 [1488105.033792] Task dump for CPU 43: [1488105.033831] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.033871] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.033874] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.033875] Task dump for CPU 64: [1488105.033878] [] ? wake_up_state+0x20/0x20 [1488105.033881] [] ? wake_up_state+0x20/0x20 [1488105.033885] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.033886] ptlrpcd_05_10 R [1488105.033887] ptlrpcd_00_06 R [1488105.033925] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.033965] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.033968] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.033969] running task [1488105.033971] 0 54226 2 0x00000088 [1488105.033973] [] ? kthread+0xd1/0xe0 [1488105.033976] [] ? kthread+0xd1/0xe0 [1488105.033980] [] apic_timer_interrupt+0x162/0x170 [1488105.033982] running task 0 54162 2 0x00000088 [1488105.033982] Call Trace: [1488105.033984] [] ? insert_kthread_work+0x40/0x40 [1488105.033988] [] ? insert_kthread_work+0x40/0x40 [1488105.033991] Call Trace: [1488105.034028] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.034031] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.034035] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.034040] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [1488105.034080] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.034118] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.034120] [] ? insert_kthread_work+0x40/0x40 [1488105.034123] [] ? insert_kthread_work+0x40/0x40 [1488105.034127] [] queued_spin_lock_slowpath+0xb/0xf [1488105.034167] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.034205] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.034206] Task dump for CPU 24: [1488105.034209] Task dump for CPU 13: [1488105.034212] [] _raw_spin_lock+0x20/0x30 [1488105.034253] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.034293] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.034296] ptlrpcd_00_11 R [1488105.034306] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.034348] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.034350] [] ? wake_up_state+0x20/0x20 [1488105.034351] ptlrpcd_06_02 R [1488105.034354] running task [1488105.034366] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.034368] [] ? wake_up_state+0x20/0x20 [1488105.034410] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.034411] 0 54167 2 0x00000088 [1488105.034415] running task 0 54230 2 0x00000088 [1488105.034453] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.034494] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.034496] [] ? kthread+0xd1/0xe0 [1488105.034497] Call Trace: [1488105.034499] Call Trace: [1488105.034537] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.034539] [] ? kthread+0xd1/0xe0 [1488105.034541] [] ? insert_kthread_work+0x40/0x40 [1488105.034578] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.034616] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.034655] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.034657] [] ? insert_kthread_work+0x40/0x40 [1488105.034660] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.034661] [] ? del_timer_sync+0x52/0x60 [1488105.034700] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.034740] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.034742] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.034744] [] ? insert_kthread_work+0x40/0x40 [1488105.034781] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.034819] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.034822] [] ? wake_up_state+0x20/0x20 [1488105.034824] [] ? insert_kthread_work+0x40/0x40 [1488105.034825] Task dump for CPU 47: [1488105.034862] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.034902] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.034941] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.034942] Task dump for CPU 69: [1488105.034982] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.034985] [] ? wake_up_state+0x20/0x20 [1488105.034989] [] kthread+0xd1/0xe0 [1488105.034989] ptlrpcd_07_02 R [1488105.034990] ptlrpcd_06_06 R [1488105.034993] [] ? wake_up_state+0x20/0x20 [1488105.035033] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.035036] [] ? insert_kthread_work+0x40/0x40 [1488105.035036] running task [1488105.035038] 0 54242 2 0x00000088 [1488105.035076] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.035079] [] ? kthread+0xd1/0xe0 [1488105.035083] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.035085] running task 0 54234 2 0x00000088 [1488105.035085] Call Trace: [1488105.035087] [] ? kthread+0xd1/0xe0 [1488105.035091] [] ? insert_kthread_work+0x40/0x40 [1488105.035095] [] ? insert_kthread_work+0x40/0x40 [1488105.035095] Call Trace: [1488105.035133] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.035136] [] ? insert_kthread_work+0x40/0x40 [1488105.035140] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.035142] Task dump for CPU 88: [1488105.035182] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.035220] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.035223] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.035226] [] ? insert_kthread_work+0x40/0x40 [1488105.035269] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.035306] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.035309] [] ? insert_kthread_work+0x40/0x40 [1488105.035311] Task dump for CPU 0: [1488105.035354] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.035396] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.035397] Task dump for CPU 26: [1488105.035400] ptlrpcd_00_00 R running task [1488105.035403] 0 54156 2 0x00000088 [1488105.035445] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.035447] [] ? wake_up_state+0x20/0x20 [1488105.035447] ptlrpcd_00_08 R [1488105.035450] ptlrpcd_01_00 R [1488105.035452] Call Trace: [1488105.035454] [] ? wake_up_state+0x20/0x20 [1488105.035495] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.035496] running task [1488105.035499] 0 54164 2 0x00000088 [1488105.035538] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.035579] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.035581] [] ? kthread+0xd1/0xe0 [1488105.035583] running task 0 54168 2 0x00000088 [1488105.035585] Call Trace: [1488105.035624] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.035626] [] ? kthread+0xd1/0xe0 [1488105.035628] [] ? insert_kthread_work+0x40/0x40 [1488105.035629] Call Trace: [1488105.035669] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.035707] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.035710] [] ? insert_kthread_work+0x40/0x40 [1488105.035712] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.035749] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.035787] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.035828] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.035830] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.035832] [] ? insert_kthread_work+0x40/0x40 [1488105.035869] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.035908] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.035911] [] ? wake_up_state+0x20/0x20 [1488105.035913] [] ? insert_kthread_work+0x40/0x40 [1488105.035914] Task dump for CPU 48: [1488105.035950] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.035991] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.036030] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.036031] Task dump for CPU 72: [1488105.036070] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.036073] [] ? wake_up_state+0x20/0x20 [1488105.036076] [] ? kthread+0xd1/0xe0 [1488105.036077] ptlrpcd_00_05 R [1488105.036079] ptlrpcd_00_04 R [1488105.036081] [] ? wake_up_state+0x20/0x20 [1488105.036121] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.036124] [] ? insert_kthread_work+0x40/0x40 [1488105.036125] running task [1488105.036126] 0 54161 2 0x00000088 [1488105.036165] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.036167] [] ? kthread+0xd1/0xe0 [1488105.036172] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.036173] running task 0 54160 2 0x00000088 [1488105.036174] Call Trace: [1488105.036176] [] ? kthread+0xd1/0xe0 [1488105.036180] [] ? insert_kthread_work+0x40/0x40 [1488105.036183] [] ? insert_kthread_work+0x40/0x40 [1488105.036183] Call Trace: [1488105.036221] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.036224] [] ? insert_kthread_work+0x40/0x40 [1488105.036228] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.036230] Task dump for CPU 95: [1488105.036271] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.036309] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.036312] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.036315] [] ? insert_kthread_work+0x40/0x40 [1488105.036358] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.036396] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.036398] [] ? insert_kthread_work+0x40/0x40 [1488105.036400] Task dump for CPU 5: [1488105.036439] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.036478] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.036479] Task dump for CPU 32: [1488105.036480] Task dump for CPU 16: [1488105.036483] ptlrpcd_07_04 R running task [1488105.036487] 0 54244 2 0x00000088 [1488105.036525] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.036527] [] ? wake_up_state+0x20/0x20 [1488105.036528] ptlrpcd_06_01 R [1488105.036529] ptlrpcd_00_10 R [1488105.036531] ptlrpcd_00_07 R [1488105.036533] Call Trace: [1488105.036535] [] ? wake_up_state+0x20/0x20 [1488105.036575] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.036575] running task [1488105.036576] running task [1488105.036579] 0 54229 2 0x00000088 [1488105.036618] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.036655] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.036657] [] ? kthread+0xd1/0xe0 [1488105.036659] 0 54166 2 0x00000088 [1488105.036661] running task 0 54163 2 0x00000088 [1488105.036663] Call Trace: [1488105.036701] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.036703] [] ? kthread+0xd1/0xe0 [1488105.036705] [] ? insert_kthread_work+0x40/0x40 [1488105.036706] Call Trace: [1488105.036707] Call Trace: [1488105.036747] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.036749] [] ? insert_kthread_work+0x40/0x40 [1488105.036751] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.036788] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.036824] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.036828] [] sched_show_task+0xa8/0x110 [1488105.036867] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.036869] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.036871] [] ? insert_kthread_work+0x40/0x40 [1488105.036908] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.036945] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.036949] [] dump_cpu_task+0x39/0x70 [1488105.036952] [] ? wake_up_state+0x20/0x20 [1488105.036954] [] ? insert_kthread_work+0x40/0x40 [1488105.036955] Task dump for CPU 56: [1488105.036992] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.037028] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.037031] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.037070] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.037071] Task dump for CPU 79: [1488105.037111] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.037149] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.037152] [] rcu_check_callbacks+0x442/0x730 [1488105.037155] [] ? kthread+0xd1/0xe0 [1488105.037156] ptlrpcd_00_03 R [1488105.037157] ptlrpcd_07_00 R [1488105.037159] [] ? wake_up_state+0x20/0x20 [1488105.037161] [] ? wake_up_state+0x20/0x20 [1488105.037164] [] ? tick_sched_do_timer+0x50/0x50 [1488105.037168] [] ? insert_kthread_work+0x40/0x40 [1488105.037169] running task [1488105.037171] 0 54159 2 0x00000088 [1488105.037209] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.037247] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.037251] [] update_process_times+0x46/0x80 [1488105.037255] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.037257] running task 0 54240 2 0x00000088 [1488105.037257] Call Trace: [1488105.037259] [] ? kthread+0xd1/0xe0 [1488105.037261] [] ? kthread+0xd1/0xe0 [1488105.037265] [] tick_sched_handle+0x30/0x70 [1488105.037269] [] ? insert_kthread_work+0x40/0x40 [1488105.037269] Call Trace: [1488105.037307] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.037309] [] ? insert_kthread_work+0x40/0x40 [1488105.037310] [] ? insert_kthread_work+0x40/0x40 [1488105.037314] [] tick_sched_timer+0x39/0x80 [1488105.037352] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.037389] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.037391] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.037394] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.037398] [] __hrtimer_run_queues+0xf3/0x270 [1488105.037436] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.037473] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.037476] [] ? insert_kthread_work+0x40/0x40 [1488105.037478] [] ? insert_kthread_work+0x40/0x40 [1488105.037481] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.037520] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.037560] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.037561] Task dump for CPU 40: [1488105.037562] Task dump for CPU 19: [1488105.037566] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.037606] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.037607] [] ? wake_up_state+0x20/0x20 [1488105.037609] ptlrpcd_00_09 R [1488105.037612] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.037616] [] ? wake_up_state+0x20/0x20 [1488105.037655] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.037656] ptlrpcd_05_02 R [1488105.037657] running task [1488105.037661] [] apic_timer_interrupt+0x162/0x170 [1488105.037701] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.037702] [] ? kthread+0xd1/0xe0 [1488105.037704] 0 54165 2 0x00000088 [1488105.037706] running task 0 54218 2 0x00000088 [1488105.037711] [] ? kthread+0xd1/0xe0 [1488105.037713] [] ? insert_kthread_work+0x40/0x40 [1488105.037714] Call Trace: [1488105.037714] Call Trace: [1488105.037719] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.037723] [] ? insert_kthread_work+0x40/0x40 [1488105.037725] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.037761] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.037798] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.037801] [] queued_spin_lock_slowpath+0xb/0xf [1488105.037806] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.037807] [] ? insert_kthread_work+0x40/0x40 [1488105.037844] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.037846] [] ? del_timer_sync+0x52/0x60 [1488105.037849] [] _raw_spin_lock+0x20/0x30 [1488105.037853] [] ? insert_kthread_work+0x40/0x40 [1488105.037854] Task dump for CPU 61: [1488105.037891] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.037928] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.037938] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.037940] Task dump for CPU 80: [1488105.037979] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.038016] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.038028] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.038030] ptlrpcd_06_08 R [1488105.038032] ptlrpcd_00_01 R [1488105.038035] [] ? wake_up_state+0x20/0x20 [1488105.038072] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.038111] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.038113] running task [1488105.038115] 0 54236 2 0x00000088 [1488105.038153] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.038155] [] ? wake_up_state+0x20/0x20 [1488105.038194] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.038198] running task 0 54157 2 0x00000088 [1488105.038198] Call Trace: [1488105.038200] [] ? kthread+0xd1/0xe0 [1488105.038238] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.038277] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.038279] Call Trace: [1488105.038317] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.038319] [] ? insert_kthread_work+0x40/0x40 [1488105.038320] [] ? kthread+0xd1/0xe0 [1488105.038361] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.038399] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.038401] [] ? del_timer_sync+0x52/0x60 [1488105.038403] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.038405] [] ? insert_kthread_work+0x40/0x40 [1488105.038409] [] ? wake_up_state+0x20/0x20 [1488105.038448] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.038485] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.038487] [] ? insert_kthread_work+0x40/0x40 [1488105.038489] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.038530] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.038568] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.038607] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.038607] Task dump for CPU 42: [1488105.038609] [] ? insert_kthread_work+0x40/0x40 [1488105.038613] [] kthread+0xd1/0xe0 [1488105.038652] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.038691] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.038693] Task dump for CPU 21: [1488105.038696] [] ? insert_kthread_work+0x40/0x40 [1488105.038700] [] ? wake_up_state+0x20/0x20 [1488105.038702] [] ? wake_up_state+0x20/0x20 [1488105.038704] ptlrpcd_01_01 R running task [1488105.038708] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.038748] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.038749] 0 54169 2 0x00000088 [1488105.038787] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.038791] [] ? insert_kthread_work+0x40/0x40 [1488105.038794] [] ? kthread+0xd1/0xe0 [1488105.038795] Call Trace: [1488105.038796] [] ? kthread+0xd1/0xe0 [1488105.038799] ptlrpcd_06_11 R running task 0 54239 2 0x00000088 [1488105.038801] Task dump for CPU 8: [1488105.038805] [] ? insert_kthread_work+0x40/0x40 [1488105.038846] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.038848] [] ? insert_kthread_work+0x40/0x40 [1488105.038849] Call Trace: [1488105.038855] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.038897] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.038900] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.038937] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.038942] [] ? insert_kthread_work+0x40/0x40 [1488105.038983] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.038985] [] ? insert_kthread_work+0x40/0x40 [1488105.039022] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.039025] ptlrpcd_00_02 R running task 0 54158 2 0x00000088 [1488105.039028] Task dump for CPU 88: [1488105.039069] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.039070] Task dump for CPU 64: [1488105.039108] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.039110] Call Trace: [1488105.039115] [] ? wake_up_state+0x20/0x20 [1488105.039116] ptlrpcd_00_00 R [1488105.039155] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.039194] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.039196] ptlrpcd_00_06 R [1488105.039238] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.039238] running task [1488105.039241] [] ? wake_up_state+0x20/0x20 [1488105.039280] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.039283] 0 54156 2 0x00000088 [1488105.039285] [] ? kthread+0xd1/0xe0 [1488105.039287] running task 0 54162 2 0x00000088 [1488105.039325] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.039364] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.039365] Call Trace: [1488105.039367] [] ? insert_kthread_work+0x40/0x40 [1488105.039368] Call Trace: [1488105.039370] [] ? kthread+0xd1/0xe0 [1488105.039411] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.039450] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.039452] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.039455] [] ? insert_kthread_work+0x40/0x40 [1488105.039459] [] ? wake_up_state+0x20/0x20 [1488105.039497] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.039499] [] ? insert_kthread_work+0x40/0x40 [1488105.039501] [] sched_show_task+0xa8/0x110 [1488105.039504] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.039545] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.039583] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.039583] Task dump for CPU 43: [1488105.039585] [] dump_cpu_task+0x39/0x70 [1488105.039587] [] ? insert_kthread_work+0x40/0x40 [1488105.039591] [] ? kthread+0xd1/0xe0 [1488105.039631] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.039634] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.039635] Task dump for CPU 24: [1488105.039638] [] ? insert_kthread_work+0x40/0x40 [1488105.039642] [] ? wake_up_state+0x20/0x20 [1488105.039645] [] rcu_check_callbacks+0x442/0x730 [1488105.039646] ptlrpcd_05_10 R running task [1488105.039651] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.039690] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.039692] 0 54226 2 0x00000088 [1488105.039694] [] ? tick_sched_do_timer+0x50/0x50 [1488105.039697] [] ? insert_kthread_work+0x40/0x40 [1488105.039700] [] ? kthread+0xd1/0xe0 [1488105.039701] Call Trace: [1488105.039703] [] update_process_times+0x46/0x80 [1488105.039705] ptlrpcd_00_11 R running task 0 54167 2 0x00000088 [1488105.039708] Task dump for CPU 13: [1488105.039712] [] ? insert_kthread_work+0x40/0x40 [1488105.039751] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.039753] [] tick_sched_handle+0x30/0x70 [1488105.039754] Call Trace: [1488105.039760] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.039801] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.039803] [] tick_sched_timer+0x39/0x80 [1488105.039840] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.039845] [] ? insert_kthread_work+0x40/0x40 [1488105.039886] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.039889] [] __hrtimer_run_queues+0xf3/0x270 [1488105.039890] [] ? del_timer_sync+0x52/0x60 [1488105.039893] ptlrpcd_06_02 R running task 0 54230 2 0x00000088 [1488105.039896] Task dump for CPU 95: [1488105.039938] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.039940] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.039978] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.039979] Call Trace: [1488105.039985] [] ? wake_up_state+0x20/0x20 [1488105.039987] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.040025] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.040064] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.040108] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.040109] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.040148] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.040186] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.040189] ptlrpcd_07_04 R running task 0 54244 2 0x00000088 [1488105.040191] [] ? kthread+0xd1/0xe0 [1488105.040194] [] apic_timer_interrupt+0x162/0x170 [1488105.040196] [] ? wake_up_state+0x20/0x20 [1488105.040234] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.040236] Call Trace: [1488105.040238] [] ? insert_kthread_work+0x40/0x40 [1488105.040277] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.040317] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.040356] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.040358] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.040361] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.040363] [] ? kthread+0xd1/0xe0 [1488105.040366] [] ? wake_up_state+0x20/0x20 [1488105.040405] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.040407] [] ? insert_kthread_work+0x40/0x40 [1488105.040409] [] queued_spin_lock_slowpath+0xb/0xf [1488105.040411] [] ? insert_kthread_work+0x40/0x40 [1488105.040451] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.040490] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.040490] Task dump for CPU 47: [1488105.040492] [] _raw_spin_lock+0x20/0x30 [1488105.040495] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.040499] [] ? kthread+0xd1/0xe0 [1488105.040539] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.040547] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.040550] [] ? insert_kthread_work+0x40/0x40 [1488105.040553] [] ? insert_kthread_work+0x40/0x40 [1488105.040556] [] ? wake_up_state+0x20/0x20 [1488105.040567] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.040569] Task dump for CPU 26: [1488105.040572] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.040612] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.040614] ptlrpcd_07_02 R running task 0 54242 2 0x00000088 [1488105.040651] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.040655] [] ? insert_kthread_work+0x40/0x40 [1488105.040658] [] ? kthread+0xd1/0xe0 [1488105.040660] Call Trace: [1488105.040698] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.040699] Task dump for CPU 0: [1488105.040703] [] ? insert_kthread_work+0x40/0x40 [1488105.040740] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.040741] ptlrpcd_01_00 R running task 0 54168 2 0x00000088 [1488105.040779] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.040785] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.040822] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.040823] Call Trace: [1488105.040862] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.040867] [] ? insert_kthread_work+0x40/0x40 [1488105.040904] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.040941] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.040944] [] ? wake_up_state+0x20/0x20 [1488105.040947] ptlrpcd_00_08 R running task 0 54164 2 0x00000088 [1488105.040988] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.041025] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.041064] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.041066] Call Trace: [1488105.041070] [] ? wake_up_state+0x20/0x20 [1488105.041107] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.041109] [] kthread+0xd1/0xe0 [1488105.041150] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.041191] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.041229] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.041232] [] ? insert_kthread_work+0x40/0x40 [1488105.041273] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.041277] [] ? kthread+0xd1/0xe0 [1488105.041279] [] ? wake_up_state+0x20/0x20 [1488105.041281] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.041321] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.041324] [] ? insert_kthread_work+0x40/0x40 [1488105.041363] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.041366] [] ? insert_kthread_work+0x40/0x40 [1488105.041407] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.041411] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.041413] [] ? kthread+0xd1/0xe0 [1488105.041414] Task dump for CPU 69: [1488105.041420] [] ? wake_up_state+0x20/0x20 [1488105.041423] [] ? insert_kthread_work+0x40/0x40 [1488105.041425] [] ? insert_kthread_work+0x40/0x40 [1488105.041430] Task dump for CPU 48: [1488105.041432] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.041438] ptlrpcd_06_06 R running task [1488105.041440] [] ? insert_kthread_work+0x40/0x40 [1488105.041442] 0 54234 2 0x00000088 [1488105.041444] Task dump for CPU 32: [1488105.041447] Task dump for CPU 16: [1488105.041450] Call Trace: [1488105.041453] ptlrpcd_00_05 R running task 0 54161 2 0x00000088 [1488105.041456] ptlrpcd_00_10 R [1488105.041500] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.041501] Call Trace: [1488105.041502] ptlrpcd_00_07 R [1488105.041508] [] ? wake_up_state+0x20/0x20 [1488105.041510] running task [1488105.041551] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.041588] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.041590] 0 54166 2 0x00000088 [1488105.041631] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.041635] running task 0 54163 2 0x00000088 [1488105.041678] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.041715] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.041716] Call Trace: [1488105.041720] [] kthread+0xd1/0xe0 [1488105.041722] Call Trace: [1488105.041764] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.041802] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.041840] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.041843] [] ? insert_kthread_work+0x40/0x40 [1488105.041886] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.041890] [] ? wake_up_state+0x20/0x20 [1488105.041930] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.041967] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.041970] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.042013] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.042057] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.042059] [] ? wake_up_state+0x20/0x20 [1488105.042096] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.042099] [] ? insert_kthread_work+0x40/0x40 [1488105.042142] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.042145] [] ? kthread+0xd1/0xe0 [1488105.042184] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.042223] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.042269] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.042273] [] ? insert_kthread_work+0x40/0x40 [1488105.042274] [] ? kthread+0xd1/0xe0 [1488105.042277] [] ? wake_up_state+0x20/0x20 [1488105.042282] [] ? wake_up_state+0x20/0x20 [1488105.042286] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.042289] [] ? insert_kthread_work+0x40/0x40 [1488105.042327] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.042372] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.042375] [] ? insert_kthread_work+0x40/0x40 [1488105.042377] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.042380] [] ? kthread+0xd1/0xe0 [1488105.042385] [] ? kthread+0xd1/0xe0 [1488105.042388] Task dump for CPU 72: [1488105.042390] [] ? insert_kthread_work+0x40/0x40 [1488105.042392] [] ? insert_kthread_work+0x40/0x40 [1488105.042397] [] ? insert_kthread_work+0x40/0x40 [1488105.042400] Code: 13 48 c1 ea [1488105.042402] Task dump for CPU 56: [1488105.042404] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.042407] ptlrpcd_00_04 R [1488105.042411] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.042413] 0d [1488105.042415] running task [1488105.042417] [] ? insert_kthread_work+0x40/0x40 [1488105.042420] ptlrpcd_00_03 R [1488105.042424] [] ? insert_kthread_work+0x40/0x40 [1488105.042426] 0 54160 2 0x00000088 [1488105.042427] 48 [1488105.042428] Task dump for CPU 40: [1488105.042431] running task [1488105.042434] Task dump for CPU 19: [1488105.042436] Call Trace: [1488105.042438] 0 54159 2 0x00000088 [1488105.042439] 98 [1488105.042442] ptlrpcd_00_09 R [1488105.042445] 83 [1488105.042487] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.042488] Call Trace: [1488105.042489] ptlrpcd_05_02 R [1488105.042491] running task [1488105.042494] e2 [1488105.042536] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.042574] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.042575] 0 54165 2 0x00000088 [1488105.042578] running task [1488105.042582] 0 54218 2 0x00000088 [1488105.042624] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.042662] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.042663] Call Trace: [1488105.042666] Call Trace: [1488105.042708] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.042746] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.042783] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.042828] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.042832] [] ? wake_up_state+0x20/0x20 [1488105.042871] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.042910] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.042915] [] ? del_timer_sync+0x52/0x60 [1488105.042958] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.042960] [] ? wake_up_state+0x20/0x20 [1488105.042996] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.043041] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.043044] [] ? kthread+0xd1/0xe0 [1488105.043083] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.043121] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.043166] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.043169] [] ? insert_kthread_work+0x40/0x40 [1488105.043171] [] ? kthread+0xd1/0xe0 [1488105.043173] [] ? wake_up_state+0x20/0x20 [1488105.043219] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.043223] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.043225] [] ? insert_kthread_work+0x40/0x40 [1488105.043264] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.043269] [] ? wake_up_state+0x20/0x20 [1488105.043273] [] ? insert_kthread_work+0x40/0x40 [1488105.043275] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.043277] [] ? kthread+0xd1/0xe0 [1488105.043322] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.043325] Task dump for CPU 79: [1488105.043327] [] ? insert_kthread_work+0x40/0x40 [1488105.043329] [] ? insert_kthread_work+0x40/0x40 [1488105.043334] [] ? kthread+0xd1/0xe0 [1488105.043338] 30 48 81 c2 80 b7 01 00 48 03 [1488105.043339] Task dump for CPU 61: [1488105.043341] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.043343] ptlrpcd_07_00 R [1488105.043348] [] ? insert_kthread_work+0x40/0x40 [1488105.043349] 14 [1488105.043351] running task [1488105.043353] [] ? insert_kthread_work+0x40/0x40 [1488105.043354] ptlrpcd_06_08 R [1488105.043359] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.043362] 0 54240 2 0x00000088 [1488105.043362] c5 [1488105.043363] Task dump for CPU 42: [1488105.043366] running task [1488105.043370] [] ? insert_kthread_work+0x40/0x40 [1488105.043373] Call Trace: [1488105.043374] 0 54236 2 0x00000088 [1488105.043375] 60 [1488105.043378] ptlrpcd_01_01 R [1488105.043381] Task dump for CPU 21: [1488105.043422] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.043423] Call Trace: [1488105.043424] b9 [1488105.043427] running task [1488105.043430] 34 [1488105.043469] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.043506] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.043507] 0 54169 2 0x00000088 [1488105.043510] ptlrpcd_06_11 R [1488105.043513] 8a [1488105.043555] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.043557] [] ? del_timer_sync+0x52/0x60 [1488105.043558] Call Trace: [1488105.043560] running task [1488105.043564] 0 54239 2 0x00000088 [1488105.043604] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.043641] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.043678] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.043682] Call Trace: [1488105.043686] [] ? wake_up_state+0x20/0x20 [1488105.043724] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.043761] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.043805] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.043849] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.043887] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.043924] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.043969] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.043972] [] ? kthread+0xd1/0xe0 [1488105.043975] [] ? wake_up_state+0x20/0x20 [1488105.044013] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.044057] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.044060] [] ? insert_kthread_work+0x40/0x40 [1488105.044098] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.044101] [] ? wake_up_state+0x20/0x20 [1488105.044147] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.044151] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.044153] [] ? kthread+0xd1/0xe0 [1488105.044191] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.044196] [] ? wake_up_state+0x20/0x20 [1488105.044200] [] ? insert_kthread_work+0x40/0x40 [1488105.044202] [] ? insert_kthread_work+0x40/0x40 [1488105.044204] [] ? kthread+0xd1/0xe0 [1488105.044249] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.044251] Task dump for CPU 80: [1488105.044253] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.044256] [] ? insert_kthread_work+0x40/0x40 [1488105.044261] [] ? kthread+0xd1/0xe0 [1488105.044263] 4c 89 02 41 8b 40 08 85 c0 [1488105.044266] [] ? insert_kthread_work+0x40/0x40 [1488105.044269] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.044271] ptlrpcd_00_01 R [1488105.044275] [] ? insert_kthread_work+0x40/0x40 [1488105.044278] 75 [1488105.044279] Task dump for CPU 64: [1488105.044281] [] ? insert_kthread_work+0x40/0x40 [1488105.044284] running task [1488105.044288] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.044291] 0 54157 2 0x00000088 [1488105.044292] 0f [1488105.044294] Task dump for CPU 43: [1488105.044297] ptlrpcd_00_06 R [1488105.044301] [] ? insert_kthread_work+0x40/0x40 [1488105.044303] Call Trace: [1488105.044304] 0f [1488105.044306] running task [1488105.044308] ptlrpcd_05_10 R [1488105.044311] Task dump for CPU 24: [1488105.044353] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.044355] 0 54162 2 0x00000088 [1488105.044355] 1f [1488105.044358] running task [1488105.044361] 44 [1488105.044400] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.044401] Call Trace: [1488105.044403] 0 54226 2 0x00000088 [1488105.044405] ptlrpcd_00_11 R [1488105.044408] 00 [1488105.044448] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.044485] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.044486] Call Trace: [1488105.044489] running task [1488105.044492] 0 54167 2 0x00000088 [1488105.044533] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.044570] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.044607] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.044612] Call Trace: [1488105.044616] [] ? wake_up_state+0x20/0x20 [1488105.044653] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.044690] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.044735] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.044776] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.044814] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.044851] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.044856] [] ? del_timer_sync+0x52/0x60 [1488105.044859] [] ? kthread+0xd1/0xe0 [1488105.044863] [] ? wake_up_state+0x20/0x20 [1488105.044901] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.044944] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.044947] [] ? insert_kthread_work+0x40/0x40 [1488105.044985] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.044987] [] ? wake_up_state+0x20/0x20 [1488105.045031] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.045035] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.045037] [] ? kthread+0xd1/0xe0 [1488105.045075] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.045121] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.045124] [] ? insert_kthread_work+0x40/0x40 [1488105.045126] [] ? insert_kthread_work+0x40/0x40 [1488105.045128] [] ? kthread+0xd1/0xe0 [1488105.045133] [] ? wake_up_state+0x20/0x20 [1488105.045136] Task dump for CPU 88: [1488105.045138] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.045140] [] ? insert_kthread_work+0x40/0x40 [1488105.045185] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.045188] 00 f3 90 <41> 8b 40 08 85 c0 [1488105.045190] [] ? insert_kthread_work+0x40/0x40 [1488105.045193] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.045196] ptlrpcd_00_00 R [1488105.045199] [] ? kthread+0xd1/0xe0 [1488105.045201] 74 [1488105.045203] Task dump for CPU 69: [1488105.045205] [] ? insert_kthread_work+0x40/0x40 [1488105.045208] running task [1488105.045212] [] ? insert_kthread_work+0x40/0x40 [1488105.045215] 0 54156 2 0x00000088 [1488105.045216] f6 [1488105.045218] Task dump for CPU 47: [1488105.045220] ptlrpcd_06_06 R [1488105.045225] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.045227] Call Trace: [1488105.045228] 4d [1488105.045230] running task [1488105.045232] ptlrpcd_07_02 R [1488105.045236] [] ? insert_kthread_work+0x40/0x40 [1488105.045276] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.045278] 0 54234 2 0x00000088 [1488105.045279] 8b [1488105.045282] running task [1488105.045285] Task dump for CPU 26: [1488105.045324] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.045325] Call Trace: [1488105.045327] 0 54242 2 0x00000088 [1488105.045331] 08 4d [1488105.045370] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.045407] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.045408] Call Trace: [1488105.045410] ptlrpcd_01_00 R [1488105.045413] 85 [1488105.045455] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.045492] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.045528] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.045530] running task [1488105.045534] 0 54168 2 0x00000088 [1488105.045538] [] ? wake_up_state+0x20/0x20 [1488105.045575] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.045612] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.045616] Call Trace: [1488105.045657] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.045696] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.045734] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.045778] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.045781] [] ? kthread+0xd1/0xe0 [1488105.045784] [] ? wake_up_state+0x20/0x20 [1488105.045822] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.045867] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.045870] [] ? insert_kthread_work+0x40/0x40 [1488105.045908] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.045911] [] ? wake_up_state+0x20/0x20 [1488105.045955] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.045959] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.045961] [] ? kthread+0xd1/0xe0 [1488105.046000] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.046045] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.046049] [] ? insert_kthread_work+0x40/0x40 [1488105.046050] [] ? insert_kthread_work+0x40/0x40 [1488105.046052] [] ? kthread+0xd1/0xe0 [1488105.046058] [] ? wake_up_state+0x20/0x20 [1488105.046060] Task dump for CPU 95: [1488105.046063] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.046066] [] ? insert_kthread_work+0x40/0x40 [1488105.046111] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.046113] c9 74 04 41 0f 18 09 8b [1488105.046116] [] ? insert_kthread_work+0x40/0x40 [1488105.046118] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.046120] [1488105.046124] [] ? kthread+0xd1/0xe0 [1488105.046127] Task dump for CPU 72: [1488105.046129] [] ? insert_kthread_work+0x40/0x40 [1488105.046133] [] ? insert_kthread_work+0x40/0x40 [1488105.046136] ptlrpcd_07_04 R running task 0 54244 2 0x00000088 [1488105.046137] Task dump for CPU 48: [1488105.046142] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.046144] Call Trace: [1488105.046146] ptlrpcd_00_04 R running task [1488105.046149] [] ? insert_kthread_work+0x40/0x40 [1488105.046189] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.046191] 0 54160 2 0x00000088 [1488105.046194] Task dump for CPU 32: [1488105.046195] Task dump for CPU 16: [1488105.046234] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.046235] Call Trace: [1488105.046236] ptlrpcd_00_05 R running task 0 54161 2 0x00000088 [1488105.046239] ptlrpcd_00_10 R [1488105.046279] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.046315] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.046316] Call Trace: [1488105.046318] ptlrpcd_00_07 R [1488105.046320] running task [1488105.046360] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.046397] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.046399] running task [1488105.046402] 0 54166 2 0x00000088 [1488105.046403] 0 54163 2 0x00000088 [1488105.046407] [] ? wake_up_state+0x20/0x20 [1488105.046445] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.046448] [] sched_show_task+0xa8/0x110 [1488105.046450] Call Trace: [1488105.046451] Call Trace: [1488105.046492] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.046530] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.046532] [] dump_cpu_task+0x39/0x70 [1488105.046572] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.046609] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.046613] [] ? kthread+0xd1/0xe0 [1488105.046616] [] ? wake_up_state+0x20/0x20 [1488105.046618] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.046661] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.046698] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.046701] [] ? insert_kthread_work+0x40/0x40 [1488105.046739] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.046741] [] rcu_check_callbacks+0x442/0x730 [1488105.046784] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.046821] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.046825] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.046826] [] ? kthread+0xd1/0xe0 [1488105.046829] [] ? tick_sched_do_timer+0x50/0x50 [1488105.046873] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.046912] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.046915] [] ? insert_kthread_work+0x40/0x40 [1488105.046918] [] ? insert_kthread_work+0x40/0x40 [1488105.046920] [] update_process_times+0x46/0x80 [1488105.046924] [] ? wake_up_state+0x20/0x20 [1488105.046927] [] ? wake_up_state+0x20/0x20 [1488105.046930] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.046932] [] tick_sched_handle+0x30/0x70 [1488105.046974] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.047013] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.047016] [] ? insert_kthread_work+0x40/0x40 [1488105.047018] [] tick_sched_timer+0x39/0x80 [1488105.047022] [] ? kthread+0xd1/0xe0 [1488105.047024] [] ? kthread+0xd1/0xe0 [1488105.047027] Task dump for CPU 79: [1488105.047029] [] __hrtimer_run_queues+0xf3/0x270 [1488105.047033] [] ? insert_kthread_work+0x40/0x40 [1488105.047035] [] ? insert_kthread_work+0x40/0x40 [1488105.047041] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.047045] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.047047] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.047052] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.047056] [] ? insert_kthread_work+0x40/0x40 [1488105.047059] [] ? insert_kthread_work+0x40/0x40 [1488105.047062] ptlrpcd_07_00 R running task 0 54240 2 0x00000088 [1488105.047064] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.047067] Task dump for CPU 40: [1488105.047068] Task dump for CPU 19: [1488105.047069] Call Trace: [1488105.047072] [] apic_timer_interrupt+0x162/0x170 [1488105.047075] ptlrpcd_00_09 R [1488105.047114] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.047115] ptlrpcd_05_02 R [1488105.047117] [1488105.047120] running task [1488105.047158] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.047161] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [1488105.047164] 0 54165 2 0x00000088 [1488105.047166] running task 0 54218 2 0x00000088 [1488105.047204] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.047206] [] queued_spin_lock_slowpath+0xb/0xf [1488105.047208] Call Trace: [1488105.047209] Call Trace: [1488105.047249] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.047251] [] _raw_spin_lock+0x20/0x30 [1488105.047294] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.047332] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.047335] [] ? wake_up_state+0x20/0x20 [1488105.047343] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.047385] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.047388] [] ? del_timer_sync+0x52/0x60 [1488105.047428] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.047438] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.047480] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.047519] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.047521] [] ? kthread+0xd1/0xe0 [1488105.047558] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.047602] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.047640] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.047643] [] ? insert_kthread_work+0x40/0x40 [1488105.047680] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.047684] [] ? wake_up_state+0x20/0x20 [1488105.047722] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.047726] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.047763] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.047806] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.047809] [] ? wake_up_state+0x20/0x20 [1488105.047812] [] ? insert_kthread_work+0x40/0x40 [1488105.047850] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.047854] [] ? kthread+0xd1/0xe0 [1488105.047893] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.047895] Task dump for CPU 80: [1488105.047897] [] ? wake_up_state+0x20/0x20 [1488105.047900] [] ? insert_kthread_work+0x40/0x40 [1488105.047902] [] ? kthread+0xd1/0xe0 [1488105.047944] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.047948] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.047950] [] ? insert_kthread_work+0x40/0x40 [1488105.047954] [] kthread+0xd1/0xe0 [1488105.047958] [] ? insert_kthread_work+0x40/0x40 [1488105.047961] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.047964] ptlrpcd_00_01 R running task 0 54157 2 0x00000088 [1488105.047966] [] ? insert_kthread_work+0x40/0x40 [1488105.047969] Task dump for CPU 42: [1488105.047971] [] ? insert_kthread_work+0x40/0x40 [1488105.047973] Call Trace: [1488105.047975] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.047980] Task dump for CPU 21: [1488105.048019] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.048021] [] ? insert_kthread_work+0x40/0x40 [1488105.048025] ptlrpcd_01_01 R running task [1488105.048063] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.048065] Task dump for CPU 56: [1488105.048068] 0 54169 2 0x00000088 [1488105.048107] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.048108] ptlrpcd_06_11 R running task [1488105.048110] Call Trace: [1488105.048112] 0 54239 2 0x00000088 [1488105.048152] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.048195] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.048196] Call Trace: [1488105.048200] [] ? wake_up_state+0x20/0x20 [1488105.048202] ptlrpcd_00_03 R running task 0 54159 2 0x00000088 [1488105.048246] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.048282] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.048322] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.048323] Call Trace: [1488105.048366] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.048403] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.048406] [] ? kthread+0xd1/0xe0 [1488105.048443] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.048487] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.048525] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.048528] [] ? insert_kthread_work+0x40/0x40 [1488105.048565] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.048569] [] ? wake_up_state+0x20/0x20 [1488105.048607] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.048611] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.048648] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.048691] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.048694] [] ? wake_up_state+0x20/0x20 [1488105.048697] [] ? insert_kthread_work+0x40/0x40 [1488105.048735] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.048739] [] ? kthread+0xd1/0xe0 [1488105.048778] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.048780] Task dump for CPU 88: [1488105.048782] [] ? wake_up_state+0x20/0x20 [1488105.048786] [] ? insert_kthread_work+0x40/0x40 [1488105.048788] [] ? kthread+0xd1/0xe0 [1488105.048829] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.048833] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.048836] [] ? insert_kthread_work+0x40/0x40 [1488105.048840] [] ? kthread+0xd1/0xe0 [1488105.048844] [] ? insert_kthread_work+0x40/0x40 [1488105.048847] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.048850] ptlrpcd_00_00 R running task 0 54156 2 0x00000088 [1488105.048853] [] ? insert_kthread_work+0x40/0x40 [1488105.048855] Task dump for CPU 43: [1488105.048858] [] ? insert_kthread_work+0x40/0x40 [1488105.048860] Call Trace: [1488105.048863] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.048867] Task dump for CPU 24: [1488105.048906] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.048908] [] ? insert_kthread_work+0x40/0x40 [1488105.048911] ptlrpcd_05_10 R running task [1488105.048950] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.048951] Task dump for CPU 61: [1488105.048954] 0 54226 2 0x00000088 [1488105.048992] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.048993] ptlrpcd_00_11 R running task [1488105.048995] Call Trace: [1488105.048997] 0 54167 2 0x00000088 [1488105.049037] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.049081] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.049082] Call Trace: [1488105.049085] [] ? wake_up_state+0x20/0x20 [1488105.049088] ptlrpcd_06_08 R running task 0 54236 2 0x00000088 [1488105.049130] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.049168] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.049207] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.049208] Call Trace: [1488105.049212] [] ? del_timer_sync+0x52/0x60 [1488105.049251] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.049254] [] ? kthread+0xd1/0xe0 [1488105.049292] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.049331] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.049371] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.049374] [] ? insert_kthread_work+0x40/0x40 [1488105.049376] [] ? del_timer_sync+0x52/0x60 [1488105.049415] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.049418] [] ? wake_up_state+0x20/0x20 [1488105.049421] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.049459] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.049500] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.049538] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.049542] [] ? insert_kthread_work+0x40/0x40 [1488105.049579] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.049582] [] ? wake_up_state+0x20/0x20 [1488105.049584] [] ? kthread+0xd1/0xe0 [1488105.049587] Task dump for CPU 95: [1488105.049625] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.049667] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.049669] [] ? insert_kthread_work+0x40/0x40 [1488105.049674] [] ? wake_up_state+0x20/0x20 [1488105.049677] [] ? kthread+0xd1/0xe0 [1488105.049680] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.049721] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.049724] [] ? insert_kthread_work+0x40/0x40 [1488105.049727] [] ? insert_kthread_work+0x40/0x40 [1488105.049730] ptlrpcd_07_04 R running task 0 54244 2 0x00000088 [1488105.049732] [] ? kthread+0xd1/0xe0 [1488105.049737] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.049738] Task dump for CPU 47: [1488105.049740] Call Trace: [1488105.049743] [] ? insert_kthread_work+0x40/0x40 [1488105.049746] [] ? insert_kthread_work+0x40/0x40 [1488105.049787] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.049789] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.049791] Task dump for CPU 26: [1488105.049830] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.049832] [] ? insert_kthread_work+0x40/0x40 [1488105.049835] ptlrpcd_07_02 R running task [1488105.049838] 0 54242 2 0x00000088 [1488105.049876] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.049877] Task dump for CPU 64: [1488105.049880] Call Trace: [1488105.049920] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.049922] ptlrpcd_01_00 R running task [1488105.049925] 0 54168 2 0x00000088 [1488105.049962] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.049965] [] ? wake_up_state+0x20/0x20 [1488105.049968] Call Trace: [1488105.050006] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.050046] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.050047] ptlrpcd_00_06 R running task 0 54162 2 0x00000088 [1488105.050087] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.050125] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.050128] [] ? kthread+0xd1/0xe0 [1488105.050129] Call Trace: [1488105.050169] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.050209] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.050212] [] ? insert_kthread_work+0x40/0x40 [1488105.050250] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.050290] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.050293] [] ? wake_up_state+0x20/0x20 [1488105.050296] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.050334] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.050375] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.050414] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.050417] [] ? insert_kthread_work+0x40/0x40 [1488105.050454] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.050458] [] ? wake_up_state+0x20/0x20 [1488105.050460] [] ? kthread+0xd1/0xe0 [1488105.050501] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.050542] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.050544] [] ? insert_kthread_work+0x40/0x40 [1488105.050548] [] ? wake_up_state+0x20/0x20 [1488105.050551] [] ? kthread+0xd1/0xe0 [1488105.050553] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.050593] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.050597] [] ? insert_kthread_work+0x40/0x40 [1488105.050599] [] ? insert_kthread_work+0x40/0x40 [1488105.050603] [] ? kthread+0xd1/0xe0 [1488105.050607] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.050609] Task dump for CPU 48: [1488105.050612] [] ? insert_kthread_work+0x40/0x40 [1488105.050617] [] ? insert_kthread_work+0x40/0x40 [1488105.050622] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.050625] Task dump for CPU 32: [1488105.050627] Task dump for CPU 16: [1488105.050633] [] ? insert_kthread_work+0x40/0x40 [1488105.050635] ptlrpcd_00_05 R running task [1488105.050638] ptlrpcd_00_10 R [1488105.050642] 0 54161 2 0x00000088 [1488105.050645] Task dump for CPU 69: [1488105.050647] ptlrpcd_00_07 R [1488105.050649] running task [1488105.050651] Call Trace: [1488105.050654] running task [1488105.050658] 0 54166 2 0x00000088 [1488105.050660] 0 54163 2 0x00000088 [1488105.050699] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.050703] Call Trace: [1488105.050705] Call Trace: [1488105.050745] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.050749] ptlrpcd_06_06 R running task 0 54234 2 0x00000088 [1488105.050787] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.050825] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.050865] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.050868] Call Trace: [1488105.050907] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.050945] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.050985] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.051024] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.051062] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.051101] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.051104] [] ? wake_up_state+0x20/0x20 [1488105.051143] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.051183] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.051222] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.051262] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.051300] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.051304] [] ? wake_up_state+0x20/0x20 [1488105.051307] [] ? wake_up_state+0x20/0x20 [1488105.051310] [] ? kthread+0xd1/0xe0 [1488105.051350] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.051390] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.051430] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.051433] [] ? insert_kthread_work+0x40/0x40 [1488105.051437] [] ? wake_up_state+0x20/0x20 [1488105.051440] [] ? kthread+0xd1/0xe0 [1488105.051443] [] ? kthread+0xd1/0xe0 [1488105.051448] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.051488] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.051491] [] ? insert_kthread_work+0x40/0x40 [1488105.051494] [] ? insert_kthread_work+0x40/0x40 [1488105.051498] [] ? insert_kthread_work+0x40/0x40 [1488105.051501] [] ? kthread+0xd1/0xe0 [1488105.051506] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.051509] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.051512] Task dump for CPU 56: [1488105.051515] [] ? insert_kthread_work+0x40/0x40 [1488105.051519] [] ? insert_kthread_work+0x40/0x40 [1488105.051523] [] ? insert_kthread_work+0x40/0x40 [1488105.051529] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.051532] Task dump for CPU 40: [1488105.051534] Task dump for CPU 19: [1488105.051539] [] ? insert_kthread_work+0x40/0x40 [1488105.051541] ptlrpcd_00_03 R running task [1488105.051544] ptlrpcd_00_09 R [1488105.051547] 0 54159 2 0x00000088 [1488105.051550] Task dump for CPU 72: [1488105.051552] ptlrpcd_05_02 R [1488105.051554] running task [1488105.051556] Call Trace: [1488105.051559] running task [1488105.051563] 0 54165 2 0x00000088 [1488105.051566] 0 54218 2 0x00000088 [1488105.051604] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.051608] Call Trace: [1488105.051610] Call Trace: [1488105.051650] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.051653] ptlrpcd_00_04 R running task 0 54160 2 0x00000088 [1488105.051692] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.051731] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.051770] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.051771] Call Trace: [1488105.051810] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.051813] [] ? del_timer_sync+0x52/0x60 [1488105.051854] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.051893] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.051932] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.051971] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.051974] [] ? wake_up_state+0x20/0x20 [1488105.052013] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.052054] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.052092] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.052132] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.052171] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.052175] [] ? wake_up_state+0x20/0x20 [1488105.052214] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.052217] [] ? kthread+0xd1/0xe0 [1488105.052258] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.052298] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.052301] [] ? wake_up_state+0x20/0x20 [1488105.052304] [] ? insert_kthread_work+0x40/0x40 [1488105.052307] [] ? wake_up_state+0x20/0x20 [1488105.052311] [] ? kthread+0xd1/0xe0 [1488105.052351] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.052354] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.052395] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.052398] [] ? insert_kthread_work+0x40/0x40 [1488105.052401] [] ? kthread+0xd1/0xe0 [1488105.052405] [] ? insert_kthread_work+0x40/0x40 [1488105.052408] [] ? kthread+0xd1/0xe0 [1488105.052413] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.052416] [] ? insert_kthread_work+0x40/0x40 [1488105.052419] Task dump for CPU 61: [1488105.052423] [] ? insert_kthread_work+0x40/0x40 [1488105.052426] [] ? insert_kthread_work+0x40/0x40 [1488105.052429] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.052435] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.052438] Task dump for CPU 42: [1488105.052441] [] ? insert_kthread_work+0x40/0x40 [1488105.052447] [] ? insert_kthread_work+0x40/0x40 [1488105.052449] ptlrpcd_06_08 R running task [1488105.052452] Task dump for CPU 21: [1488105.052454] 0 54236 2 0x00000088 [1488105.052457] Task dump for CPU 79: [1488105.052461] ptlrpcd_01_01 R running task [1488105.052463] Call Trace: [1488105.052466] ptlrpcd_06_11 R [1488105.052469] 0 54169 2 0x00000088 [1488105.052471] ptlrpcd_07_00 R [1488105.052510] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.052512] running task [1488105.052514] Call Trace: [1488105.052517] 0 54239 2 0x00000088 [1488105.052521] [] ? del_timer_sync+0x52/0x60 [1488105.052524] running task 0 54240 2 0x00000088 [1488105.052563] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.052565] Call Trace: [1488105.052605] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.052607] Call Trace: [1488105.052646] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.052684] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.052723] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.052762] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.052800] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.052839] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.052880] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.052919] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.052958] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.052997] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.053000] [] ? wake_up_state+0x20/0x20 [1488105.053038] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.053041] [] ? wake_up_state+0x20/0x20 [1488105.053081] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.053121] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.053161] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.053201] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.053204] [] ? wake_up_state+0x20/0x20 [1488105.053206] [] ? kthread+0xd1/0xe0 [1488105.053210] [] ? wake_up_state+0x20/0x20 [1488105.053213] [] ? kthread+0xd1/0xe0 [1488105.053254] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.053257] [] ? insert_kthread_work+0x40/0x40 [1488105.053297] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.053300] [] ? insert_kthread_work+0x40/0x40 [1488105.053303] [] ? kthread+0xd1/0xe0 [1488105.053306] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.053310] [] ? kthread+0xd1/0xe0 [1488105.053314] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.053318] [] ? insert_kthread_work+0x40/0x40 [1488105.053321] [] ? insert_kthread_work+0x40/0x40 [1488105.053324] [] ? insert_kthread_work+0x40/0x40 [1488105.053328] [] ? insert_kthread_work+0x40/0x40 [1488105.053331] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.053336] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.053338] Task dump for CPU 43: [1488105.053342] [] ? insert_kthread_work+0x40/0x40 [1488105.053345] [] ? insert_kthread_work+0x40/0x40 [1488105.053351] Task dump for CPU 24: [1488105.053353] Task dump for CPU 80: [1488105.053356] Task dump for CPU 64: [1488105.053359] ptlrpcd_05_10 R running task [1488105.053361] ptlrpcd_00_11 R [1488105.053364] ptlrpcd_00_01 R [1488105.053365] 0 54226 2 0x00000088 [1488105.053367] ptlrpcd_00_06 R [1488105.053370] running task [1488105.053372] running task [1488105.053373] Call Trace: [1488105.053376] 0 54167 2 0x00000088 [1488105.053379] 0 54157 2 0x00000088 [1488105.053382] running task 0 54162 2 0x00000088 [1488105.053419] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.053421] Call Trace: [1488105.053423] Call Trace: [1488105.053426] Call Trace: [1488105.053464] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.053502] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.053541] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.053583] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.053621] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.053624] [] ? del_timer_sync+0x52/0x60 [1488105.053663] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.053706] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.053745] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.053783] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.053821] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.053864] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.053867] [] ? wake_up_state+0x20/0x20 [1488105.053905] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.053945] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.053988] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.054027] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.054067] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.054070] [] ? wake_up_state+0x20/0x20 [1488105.054074] [] ? wake_up_state+0x20/0x20 [1488105.054076] [] ? kthread+0xd1/0xe0 [1488105.054081] [] ? wake_up_state+0x20/0x20 [1488105.054120] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.054164] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.054166] [] ? insert_kthread_work+0x40/0x40 [1488105.054206] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.054210] [] ? kthread+0xd1/0xe0 [1488105.054213] [] ? kthread+0xd1/0xe0 [1488105.054216] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.054219] [] ? kthread+0xd1/0xe0 [1488105.054222] [] ? insert_kthread_work+0x40/0x40 [1488105.054226] [] ? insert_kthread_work+0x40/0x40 [1488105.054228] [] ? insert_kthread_work+0x40/0x40 [1488105.054231] [] ? insert_kthread_work+0x40/0x40 [1488105.054235] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.054239] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.054240] Task dump for CPU 47: [1488105.054245] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.054249] [] ? insert_kthread_work+0x40/0x40 [1488105.054253] [] ? insert_kthread_work+0x40/0x40 [1488105.054258] [] ? insert_kthread_work+0x40/0x40 [1488105.054261] Task dump for CPU 88: [1488105.054263] Task dump for CPU 69: [1488105.054266] Task dump for CPU 26: [1488105.054268] ptlrpcd_07_02 R running task [1488105.054271] ptlrpcd_00_00 R [1488105.054273] 0 54242 2 0x00000088 [1488105.054275] ptlrpcd_06_06 R [1488105.054277] ptlrpcd_01_00 R [1488105.054280] running task [1488105.054281] Call Trace: [1488105.054283] running task [1488105.054287] 0 54156 2 0x00000088 [1488105.054289] 0 54234 2 0x00000088 [1488105.054327] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.054330] running task 0 54168 2 0x00000088 [1488105.054332] Call Trace: [1488105.054335] Call Trace: [1488105.054372] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.054374] Call Trace: [1488105.054413] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.054456] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.054494] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.054533] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.054571] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.054614] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.054653] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.054692] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.054730] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.054772] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.054775] [] ? wake_up_state+0x20/0x20 [1488105.054814] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.054853] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.054897] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.054936] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.054976] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.054980] [] ? wake_up_state+0x20/0x20 [1488105.054983] [] ? wake_up_state+0x20/0x20 [1488105.054985] [] ? kthread+0xd1/0xe0 [1488105.054989] [] ? wake_up_state+0x20/0x20 [1488105.055029] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.055073] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.055074] [] ? insert_kthread_work+0x40/0x40 [1488105.055114] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.055117] [] ? kthread+0xd1/0xe0 [1488105.055121] [] ? kthread+0xd1/0xe0 [1488105.055124] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.055127] [] ? kthread+0xd1/0xe0 [1488105.055131] [] ? insert_kthread_work+0x40/0x40 [1488105.055134] [] ? insert_kthread_work+0x40/0x40 [1488105.055136] [] ? insert_kthread_work+0x40/0x40 [1488105.055140] [] ? insert_kthread_work+0x40/0x40 [1488105.055144] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.055148] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.055149] Task dump for CPU 48: [1488105.055153] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.055156] [] ? insert_kthread_work+0x40/0x40 [1488105.055159] [] ? insert_kthread_work+0x40/0x40 [1488105.055164] [] ? insert_kthread_work+0x40/0x40 [1488105.055167] Task dump for CPU 95: [1488105.055169] Task dump for CPU 72: [1488105.055172] Task dump for CPU 32: [1488105.055175] ptlrpcd_00_05 R running task [1488105.055178] ptlrpcd_07_04 R [1488105.055179] 0 54161 2 0x00000088 [1488105.055182] ptlrpcd_00_04 R [1488105.055184] ptlrpcd_00_10 R [1488105.055186] running task [1488105.055187] Call Trace: [1488105.055188] running task [1488105.055192] 0 54244 2 0x00000088 [1488105.055195] 0 54160 2 0x00000088 [1488105.055232] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.055235] running task 0 54166 2 0x00000088 [1488105.055237] Call Trace: [1488105.055239] Call Trace: [1488105.055278] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.055280] Call Trace: [1488105.055319] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.055359] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.055397] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.055436] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.055474] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.055517] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.055557] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.055595] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.055633] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.055676] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.055679] [] ? wake_up_state+0x20/0x20 [1488105.055718] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.055757] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.055801] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.055839] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.055879] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.055882] [] ? wake_up_state+0x20/0x20 [1488105.055886] [] ? wake_up_state+0x20/0x20 [1488105.055889] [] ? kthread+0xd1/0xe0 [1488105.055893] [] ? wake_up_state+0x20/0x20 [1488105.055932] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.055976] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.055977] [] ? insert_kthread_work+0x40/0x40 [1488105.056017] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.056020] [] ? kthread+0xd1/0xe0 [1488105.056024] [] ? kthread+0xd1/0xe0 [1488105.056027] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056030] [] ? kthread+0xd1/0xe0 [1488105.056034] [] ? insert_kthread_work+0x40/0x40 [1488105.056038] [] ? insert_kthread_work+0x40/0x40 [1488105.056040] [] ? insert_kthread_work+0x40/0x40 [1488105.056043] [] ? insert_kthread_work+0x40/0x40 [1488105.056047] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056051] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056052] Task dump for CPU 56: [1488105.056055] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056059] [] ? insert_kthread_work+0x40/0x40 [1488105.056063] [] ? insert_kthread_work+0x40/0x40 [1488105.056067] [] ? insert_kthread_work+0x40/0x40 [1488105.056069] Task dump for CPU 79: [1488105.056072] Task dump for CPU 40: [1488105.056074] ptlrpcd_00_03 R running task [1488105.056075] 0 54159 2 0x00000088 [1488105.056078] ptlrpcd_07_00 R [1488105.056080] ptlrpcd_00_09 R [1488105.056081] Call Trace: [1488105.056083] running task [1488105.056087] 0 54240 2 0x00000088 [1488105.056125] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.056128] running task 0 54165 2 0x00000088 [1488105.056130] Call Trace: [1488105.056167] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.056169] Call Trace: [1488105.056211] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.056249] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.056287] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.056330] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.056369] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.056408] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.056450] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.056452] [] ? wake_up_state+0x20/0x20 [1488105.056491] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.056534] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.056573] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.056613] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.056616] [] ? wake_up_state+0x20/0x20 [1488105.056618] [] ? kthread+0xd1/0xe0 [1488105.056621] [] ? wake_up_state+0x20/0x20 [1488105.056663] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.056665] [] ? insert_kthread_work+0x40/0x40 [1488105.056705] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.056708] [] ? kthread+0xd1/0xe0 [1488105.056710] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056713] [] ? kthread+0xd1/0xe0 [1488105.056716] [] ? insert_kthread_work+0x40/0x40 [1488105.056718] [] ? insert_kthread_work+0x40/0x40 [1488105.056722] [] ? insert_kthread_work+0x40/0x40 [1488105.056725] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056726] Task dump for CPU 61: [1488105.056730] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.056733] [] ? insert_kthread_work+0x40/0x40 [1488105.056737] [] ? insert_kthread_work+0x40/0x40 [1488105.056739] Task dump for CPU 80: [1488105.056742] Task dump for CPU 42: [1488105.056745] ptlrpcd_06_08 R running task [1488105.056747] 0 54236 2 0x00000088 [1488105.056750] ptlrpcd_00_01 R [1488105.056752] ptlrpcd_01_01 R [1488105.056753] Call Trace: [1488105.056755] running task [1488105.056759] 0 54157 2 0x00000088 [1488105.056760] running task [1488105.056763] 0 54169 2 0x00000088 [1488105.056764] Call Trace: [1488105.056767] [] sched_show_task+0xa8/0x110 [1488105.056769] Call Trace: [1488105.056812] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.056815] [] dump_cpu_task+0x39/0x70 [1488105.056853] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.056896] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.056898] [] rcu_dump_cpu_stacks+0x90/0xd0 [1488105.056937] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.056980] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.056982] [] rcu_check_callbacks+0x442/0x730 [1488105.057020] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.057063] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.057065] [] ? tick_sched_do_timer+0x50/0x50 [1488105.057105] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.057108] [] ? wake_up_state+0x20/0x20 [1488105.057110] [] update_process_times+0x46/0x80 [1488105.057114] [] ? wake_up_state+0x20/0x20 [1488105.057157] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.057158] [] tick_sched_handle+0x30/0x70 [1488105.057198] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.057202] [] ? kthread+0xd1/0xe0 [1488105.057204] [] tick_sched_timer+0x39/0x80 [1488105.057207] [] ? kthread+0xd1/0xe0 [1488105.057211] [] ? insert_kthread_work+0x40/0x40 [1488105.057213] [] __hrtimer_run_queues+0xf3/0x270 [1488105.057217] [] ? insert_kthread_work+0x40/0x40 [1488105.057220] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.057222] [] hrtimer_interrupt+0xaf/0x1d0 [1488105.057226] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.057229] [] ? insert_kthread_work+0x40/0x40 [1488105.057232] [] local_apic_timer_interrupt+0x3b/0x60 [1488105.057235] [] ? insert_kthread_work+0x40/0x40 [1488105.057238] Task dump for CPU 88: [1488105.057240] [] smp_apic_timer_interrupt+0x43/0x60 [1488105.057242] Task dump for CPU 43: [1488105.057248] [] apic_timer_interrupt+0x162/0x170 [1488105.057250] ptlrpcd_00_00 R [1488105.057252] ptlrpcd_05_10 R [1488105.057253] running task [1488105.057256] [1488105.057259] 0 54156 2 0x00000088 [1488105.057262] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488105.057265] running task 0 54226 2 0x00000088 [1488105.057267] Call Trace: [1488105.057269] [] queued_spin_lock_slowpath+0xb/0xf [1488105.057271] Call Trace: [1488105.057314] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.057317] [] _raw_spin_lock+0x20/0x30 [1488105.057356] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.057398] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.057407] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.057445] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.057487] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.057498] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.057536] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.057580] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.057618] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.057657] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.057661] [] ? wake_up_state+0x20/0x20 [1488105.057663] [] ? del_timer_sync+0x52/0x60 [1488105.057667] [] ? wake_up_state+0x20/0x20 [1488105.057710] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.057748] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.057787] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.057790] [] ? kthread+0xd1/0xe0 [1488105.057827] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.057830] [] ? kthread+0xd1/0xe0 [1488105.057833] [] ? insert_kthread_work+0x40/0x40 [1488105.057872] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.057875] [] ? insert_kthread_work+0x40/0x40 [1488105.057879] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.057881] [] ? wake_up_state+0x20/0x20 [1488105.057884] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.057888] [] ? insert_kthread_work+0x40/0x40 [1488105.057926] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.057930] [] ? insert_kthread_work+0x40/0x40 [1488105.057932] Task dump for CPU 95: [1488105.057934] [] kthread+0xd1/0xe0 [1488105.057937] Task dump for CPU 47: [1488105.057941] [] ? insert_kthread_work+0x40/0x40 [1488105.057943] ptlrpcd_07_04 R [1488105.057945] ptlrpcd_07_02 R [1488105.057948] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.057950] running task [1488105.057953] 0 54244 2 0x00000088 [1488105.057955] [] ? insert_kthread_work+0x40/0x40 [1488105.057958] running task 0 54242 2 0x00000088 [1488105.057961] Call Trace: [1488105.057962] Task dump for CPU 64: [1488105.057964] Call Trace: [1488105.058006] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.058046] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.058085] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.058124] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.058127] ptlrpcd_00_06 R running task 0 54162 2 0x00000088 [1488105.058164] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.058202] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.058204] Call Trace: [1488105.058243] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.058283] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.058323] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.058326] [] ? wake_up_state+0x20/0x20 [1488105.058329] [] ? wake_up_state+0x20/0x20 [1488105.058368] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.058407] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.058446] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.058486] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.058487] [] ? kthread+0xd1/0xe0 [1488105.058491] [] ? kthread+0xd1/0xe0 [1488105.058532] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.058534] [] ? insert_kthread_work+0x40/0x40 [1488105.058537] [] ? insert_kthread_work+0x40/0x40 [1488105.058540] [] ? wake_up_state+0x20/0x20 [1488105.058543] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.058546] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.058587] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.058590] [] ? insert_kthread_work+0x40/0x40 [1488105.058593] [] ? insert_kthread_work+0x40/0x40 [1488105.058596] [] ? kthread+0xd1/0xe0 [1488105.058599] Task dump for CPU 48: [1488105.058603] [] ? insert_kthread_work+0x40/0x40 [1488105.058610] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.058614] [] ? insert_kthread_work+0x40/0x40 [1488105.058617] ptlrpcd_00_05 R running task 0 54161 2 0x00000088 [1488105.058619] Task dump for CPU 69: [1488105.058620] Call Trace: [1488105.058662] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.058701] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.058703] ptlrpcd_06_06 R running task 0 54234 2 0x00000088 [1488105.058742] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.058743] Call Trace: [1488105.058783] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.058821] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.058824] [] ? wake_up_state+0x20/0x20 [1488105.058863] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.058903] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.058942] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.058945] [] ? kthread+0xd1/0xe0 [1488105.058985] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.058988] [] ? insert_kthread_work+0x40/0x40 [1488105.058991] [] ? wake_up_state+0x20/0x20 [1488105.058994] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.059033] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.059036] [] ? insert_kthread_work+0x40/0x40 [1488105.059039] [] ? kthread+0xd1/0xe0 [1488105.059041] Task dump for CPU 56: [1488105.059045] [] ? insert_kthread_work+0x40/0x40 [1488105.059050] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.059055] [] ? insert_kthread_work+0x40/0x40 [1488105.059057] ptlrpcd_00_03 R running task 0 54159 2 0x00000088 [1488105.059060] Task dump for CPU 72: [1488105.059062] Call Trace: [1488105.059103] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.059105] ptlrpcd_00_04 R running task 0 54160 2 0x00000088 [1488105.059144] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.059146] Call Trace: [1488105.059184] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.059222] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.059263] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.059301] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.059304] [] ? wake_up_state+0x20/0x20 [1488105.059343] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.059382] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.059422] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.059425] [] ? kthread+0xd1/0xe0 [1488105.059428] [] ? wake_up_state+0x20/0x20 [1488105.059431] [] ? insert_kthread_work+0x40/0x40 [1488105.059470] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.059474] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.059476] [] ? kthread+0xd1/0xe0 [1488105.059479] [] ? insert_kthread_work+0x40/0x40 [1488105.059482] [] ? insert_kthread_work+0x40/0x40 [1488105.059485] Task dump for CPU 61: [1488105.059488] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.059493] [] ? insert_kthread_work+0x40/0x40 [1488105.059496] Task dump for CPU 79: [1488105.059498] ptlrpcd_06_08 R running task 0 54236 2 0x00000088 [1488105.059502] Call Trace: [1488105.059543] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.059545] ptlrpcd_07_00 R running task 0 54240 2 0x00000088 [1488105.059549] [] ? del_timer_sync+0x52/0x60 [1488105.059551] Call Trace: [1488105.059589] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.059628] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.059665] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.059705] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.059744] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.059782] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.059785] [] ? wake_up_state+0x20/0x20 [1488105.059825] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.059864] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.059867] [] ? wake_up_state+0x20/0x20 [1488105.059870] [] ? kthread+0xd1/0xe0 [1488105.059910] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.059913] [] ? insert_kthread_work+0x40/0x40 [1488105.059916] [] ? kthread+0xd1/0xe0 [1488105.059920] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.059923] [] ? insert_kthread_work+0x40/0x40 [1488105.059926] [] ? insert_kthread_work+0x40/0x40 [1488105.059930] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.059931] [] ? insert_kthread_work+0x40/0x40 [1488105.059933] Task dump for CPU 80: [1488105.059937] Task dump for CPU 64: [1488105.059940] ptlrpcd_00_01 R running task [1488105.059944] 0 54157 2 0x00000088 [1488105.059947] Call Trace: [1488105.059950] ptlrpcd_00_06 R running task 0 54162 2 0x00000088 [1488105.059989] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.059991] Call Trace: [1488105.060030] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.060069] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.060107] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.060146] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.060186] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.060224] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.060226] [] ? wake_up_state+0x20/0x20 [1488105.060267] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.060306] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.060309] [] ? wake_up_state+0x20/0x20 [1488105.060312] [] ? kthread+0xd1/0xe0 [1488105.060352] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.060355] [] ? insert_kthread_work+0x40/0x40 [1488105.060358] [] ? kthread+0xd1/0xe0 [1488105.060361] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.060365] [] ? insert_kthread_work+0x40/0x40 [1488105.060368] [] ? insert_kthread_work+0x40/0x40 [1488105.060371] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.060374] Task dump for CPU 88: [1488105.060377] [] ? insert_kthread_work+0x40/0x40 [1488105.060382] Task dump for CPU 69: [1488105.060386] ptlrpcd_00_00 R running task [1488105.060389] 0 54156 2 0x00000088 [1488105.060392] Call Trace: [1488105.060395] ptlrpcd_06_06 R running task 0 54234 2 0x00000088 [1488105.060434] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.060435] Call Trace: [1488105.060474] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.060513] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.060551] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.060590] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.060629] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.060667] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.060670] [] ? wake_up_state+0x20/0x20 [1488105.060710] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.060749] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.060752] [] ? wake_up_state+0x20/0x20 [1488105.060755] [] ? kthread+0xd1/0xe0 [1488105.060794] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.060798] [] ? insert_kthread_work+0x40/0x40 [1488105.060801] [] ? kthread+0xd1/0xe0 [1488105.060805] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.060808] [] ? insert_kthread_work+0x40/0x40 [1488105.060811] [] ? insert_kthread_work+0x40/0x40 [1488105.060814] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.060816] Task dump for CPU 95: [1488105.060820] [] ? insert_kthread_work+0x40/0x40 [1488105.060824] Task dump for CPU 72: [1488105.060827] ptlrpcd_07_04 R running task [1488105.060830] 0 54244 2 0x00000088 [1488105.060834] Call Trace: [1488105.060836] ptlrpcd_00_04 R running task 0 54160 2 0x00000088 [1488105.060876] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.060877] Call Trace: [1488105.060916] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.060955] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.060993] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.061034] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.061074] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.061112] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.061116] [] ? wake_up_state+0x20/0x20 [1488105.061156] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.061196] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.061199] [] ? wake_up_state+0x20/0x20 [1488105.061202] [] ? kthread+0xd1/0xe0 [1488105.061241] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.061244] [] ? insert_kthread_work+0x40/0x40 [1488105.061247] [] ? kthread+0xd1/0xe0 [1488105.061251] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.061254] [] ? insert_kthread_work+0x40/0x40 [1488105.061257] [] ? insert_kthread_work+0x40/0x40 [1488105.061260] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.061262] [] ? insert_kthread_work+0x40/0x40 [1488105.061263] Task dump for CPU 79: [1488105.061265] ptlrpcd_07_00 R running task 0 54240 2 0x00000088 [1488105.061266] Call Trace: [1488105.061304] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.061342] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.061379] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.061418] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.061420] [] ? wake_up_state+0x20/0x20 [1488105.061458] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.061460] [] ? kthread+0xd1/0xe0 [1488105.061462] [] ? insert_kthread_work+0x40/0x40 [1488105.061464] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.061466] [] ? insert_kthread_work+0x40/0x40 [1488105.061467] Task dump for CPU 80: [1488105.061469] ptlrpcd_00_01 R running task 0 54157 2 0x00000088 [1488105.061470] Call Trace: [1488105.061508] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.061546] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.061582] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.061621] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.061622] [] ? wake_up_state+0x20/0x20 [1488105.061661] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.061662] [] ? kthread+0xd1/0xe0 [1488105.061664] [] ? insert_kthread_work+0x40/0x40 [1488105.061666] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.061667] [] ? insert_kthread_work+0x40/0x40 [1488105.061668] Task dump for CPU 88: [1488105.061670] ptlrpcd_00_00 R running task 0 54156 2 0x00000088 [1488105.061670] Call Trace: [1488105.061707] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.061745] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.061782] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.061820] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.061822] [] ? wake_up_state+0x20/0x20 [1488105.061860] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.061861] [] ? kthread+0xd1/0xe0 [1488105.061863] [] ? insert_kthread_work+0x40/0x40 [1488105.061865] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.061866] [] ? insert_kthread_work+0x40/0x40 [1488105.061867] Task dump for CPU 95: [1488105.061869] ptlrpcd_07_04 R running task 0 54244 2 0x00000088 [1488105.061869] Call Trace: [1488105.061906] [] ? ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.061944] [] ? ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.061981] [] ? ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.062020] [] ? ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.062021] [] ? wake_up_state+0x20/0x20 [1488105.062061] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.062063] [] ? kthread+0xd1/0xe0 [1488105.062064] [] ? insert_kthread_work+0x40/0x40 [1488105.062066] [] ? ret_from_fork_nospec_begin+0xe/0x21 [1488105.062067] [] ? insert_kthread_work+0x40/0x40 [1488105.240260] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ptlrpcd_05_08:54224] [1488105.240290] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488105.240298] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488105.240301] CPU: 11 PID: 54224 Comm: ptlrpcd_05_08 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488105.240302] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488105.240303] task: ffff8abafb339040 ti: ffff8abaeca28000 task.ti: ffff8abaeca28000 [1488105.240306] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488105.240307] RSP: 0018:ffff8abaeca2bb78 EFLAGS: 00000246 [1488105.240308] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000590000 [1488105.240309] RDX: ffff8abaffc1b780 RSI: 0000000002310001 RDI: ffff8adafa7b1b00 [1488105.240311] RBP: ffff8abaeca2bb78 R08: ffff8adaffa5b780 R09: 0000000000000000 [1488105.240312] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad89cf94000 [1488105.240313] R13: 0005c4aa17a3b130 R14: ffff8ad165ebe300 R15: 0000000000000007 [1488105.240315] FS: 00007ea1b7062700(0000) GS:ffff8adaffa40000(0000) knlGS:0000000000000000 [1488105.240316] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488105.240317] CR2: 00007fbad3d83000 CR3: 0000002034d8c000 CR4: 00000000003407e0 [1488105.240317] Call Trace: [1488105.240320] [] queued_spin_lock_slowpath+0xb/0xf [1488105.240322] [] _raw_spin_lock+0x20/0x30 [1488105.240330] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488105.240341] [] LNetMDUnlink+0x78/0x180 [lnet] [1488105.240379] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488105.240382] [] ? del_timer_sync+0x52/0x60 [1488105.240419] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488105.240457] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488105.240496] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488105.240498] [] ? wake_up_state+0x20/0x20 [1488105.240537] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488105.240539] [] kthread+0xd1/0xe0 [1488105.240541] [] ? insert_kthread_work+0x40/0x40 [1488105.240543] [] ret_from_fork_nospec_begin+0xe/0x21 [1488105.240545] [] ? insert_kthread_work+0x40/0x40 [1488105.240565] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488113.167781] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ptlrpcd_05_01:54217] [1488113.167809] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.167817] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.167819] CPU: 3 PID: 54217 Comm: ptlrpcd_05_01 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.167820] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.167821] task: ffff8abaee639040 ti: ffff8abafaf6c000 task.ti: ffff8abafaf6c000 [1488113.167824] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488113.167825] RSP: 0018:ffff8abafaf6fb78 EFLAGS: 00000246 [1488113.167826] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000190000 [1488113.167827] RDX: ffff8a8aff49b780 RSI: 0000000002810001 RDI: ffff8adafa7b1b00 [1488113.167828] RBP: ffff8abafaf6fb78 R08: ffff8adaffa1b780 R09: 0000000000000000 [1488113.167829] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad1c9f1aa00 [1488113.167830] R13: 0005c4aa17afa680 R14: ffff8adab82b8300 R15: 0000000000000007 [1488113.167832] FS: 00007ea1d2098700(0000) GS:ffff8adaffa00000(0000) knlGS:0000000000000000 [1488113.167833] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.167834] CR2: 00007fff8047d958 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.167835] Call Trace: [1488113.167837] [] queued_spin_lock_slowpath+0xb/0xf [1488113.167839] [] _raw_spin_lock+0x20/0x30 [1488113.167847] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.167858] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.167896] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.167934] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.167971] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.168010] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.168013] [] ? wake_up_state+0x20/0x20 [1488113.168051] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.168053] [] kthread+0xd1/0xe0 [1488113.168055] [] ? insert_kthread_work+0x40/0x40 [1488113.168057] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.168059] [] ? insert_kthread_work+0x40/0x40 [1488113.168079] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488113.385794] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 22s! [ptlrpcd_05_06:54222] [1488113.385821] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.385829] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.385832] CPU: 27 PID: 54222 Comm: ptlrpcd_05_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.385832] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.385834] task: ffff8abaee63e180 ti: ffff8abafdbdc000 task.ti: ffff8abafdbdc000 [1488113.385837] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.385838] RSP: 0018:ffff8abafdbdfb78 EFLAGS: 00000246 [1488113.385839] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000d90000 [1488113.385840] RDX: ffff8acaffa9b780 RSI: 0000000000890001 RDI: ffff8adafa7b1b00 [1488113.385841] RBP: ffff8abafdbdfb78 R08: ffff8adaffadb780 R09: 0000000000000000 [1488113.385842] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ad8b63d6600 [1488113.385843] R13: 0005c4aa17d33040 R14: ffff8ad23dd66c00 R15: 0000000000000007 [1488113.385844] FS: 00007f1993824740(0000) GS:ffff8adaffac0000(0000) knlGS:0000000000000000 [1488113.385845] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.385847] CR2: 00007f8c1d455000 CR3: 0000005f8433a000 CR4: 00000000003407e0 [1488113.385847] Call Trace: [1488113.385849] [] queued_spin_lock_slowpath+0xb/0xf [1488113.385852] [] _raw_spin_lock+0x20/0x30 [1488113.385860] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.385870] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.385909] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.385947] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.385985] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.386024] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.386026] [] ? wake_up_state+0x20/0x20 [1488113.386065] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.386067] [] kthread+0xd1/0xe0 [1488113.386069] [] ? insert_kthread_work+0x40/0x40 [1488113.386071] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.386073] [] ? insert_kthread_work+0x40/0x40 [1488113.386093] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.404795] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [ptlrpcd_06_10:54238] [1488113.404821] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.404829] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.404831] CPU: 29 PID: 54238 Comm: ptlrpcd_06_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.404832] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.404834] task: ffff8abafbc79040 ti: ffff8abaf8e64000 task.ti: ffff8abaf8e64000 [1488113.404837] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.404837] RSP: 0018:ffff8abaf8e67b78 EFLAGS: 00000246 [1488113.404838] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000e90000 [1488113.404840] RDX: ffff8aeaffa5b780 RSI: 0000000000690001 RDI: ffff8adafa7b1b00 [1488113.404841] RBP: ffff8abaf8e67b78 R08: ffff8aeaffadb780 R09: 0000000000000000 [1488113.404841] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adbe028a600 [1488113.404842] R13: 0005c4aa17c61800 R14: ffff8ae33a3d4e00 R15: 0000000000000007 [1488113.404844] FS: 00007ea1b485d700(0000) GS:ffff8aeaffac0000(0000) knlGS:0000000000000000 [1488113.404845] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.404846] CR2: 00007f470eefd038 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.404847] Call Trace: [1488113.404849] [] queued_spin_lock_slowpath+0xb/0xf [1488113.404851] [] _raw_spin_lock+0x20/0x30 [1488113.404859] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.404870] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.404908] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.404946] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.404983] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.405021] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.405024] [] ? wake_up_state+0x20/0x20 [1488113.405062] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.405064] [] kthread+0xd1/0xe0 [1488113.405066] [] ? insert_kthread_work+0x40/0x40 [1488113.405068] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.405070] [] ? insert_kthread_work+0x40/0x40 [1488113.405090] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.550805] NMI watchdog: BUG: soft lockup - CPU#45 stuck for 22s! [ptlrpcd_06_00:54228] [1488113.550831] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.550838] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.550841] CPU: 45 PID: 54228 Comm: ptlrpcd_06_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.550842] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.550843] task: ffff8abafb33d140 ti: ffff8abafc584000 task.ti: ffff8abafc584000 [1488113.550846] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488113.550847] RSP: 0018:ffff8abafc587b78 EFLAGS: 00000246 [1488113.550848] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001690000 [1488113.550849] RDX: ffff8afb3f81b780 RSI: 0000000000390001 RDI: ffff8adafa7b1b00 [1488113.550850] RBP: ffff8abafc587b78 R08: ffff8aeaffb5b780 R09: 0000000000000000 [1488113.550851] R10: 0000000000000000 R11: 000000000000000f R12: ffff8adc55a5de00 [1488113.550852] R13: 0005c4aa17c41470 R14: ffff8ae89e599500 R15: 0000000000000007 [1488113.550853] FS: 00007ea1b485d700(0000) GS:ffff8aeaffb40000(0000) knlGS:0000000000000000 [1488113.550854] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.550856] CR2: 00007eea0371e002 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.550856] Call Trace: [1488113.550859] [] queued_spin_lock_slowpath+0xb/0xf [1488113.550861] [] _raw_spin_lock+0x20/0x30 [1488113.550869] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.550880] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.550918] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.550956] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.550994] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.551033] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.551035] [] ? wake_up_state+0x20/0x20 [1488113.551075] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.551076] [] kthread+0xd1/0xe0 [1488113.551078] [] ? insert_kthread_work+0x40/0x40 [1488113.551080] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.551082] [] ? insert_kthread_work+0x40/0x40 [1488113.551102] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488113.595808] NMI watchdog: BUG: soft lockup - CPU#50 stuck for 22s! [ptlrpcd_01_02:54170] [1488113.595837] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.595844] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.595847] CPU: 50 PID: 54170 Comm: ptlrpcd_01_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.595848] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.595849] task: ffff8abaf63430c0 ti: ffff8abafa204000 task.ti: ffff8abafa204000 [1488113.595852] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.595853] RSP: 0018:ffff8abafa207b78 EFLAGS: 00000246 [1488113.595854] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001910000 [1488113.595855] RDX: ffff8aeaffb5b780 RSI: 0000000001690000 RDI: ffff8adafa7b1b00 [1488113.595856] RBP: ffff8abafa207b78 R08: ffff8a9affb9b780 R09: 0000000000000000 [1488113.595857] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a92577cf600 [1488113.595858] R13: 0005c4aa179a0170 R14: ffff8a9280b56600 R15: 0000000000000007 [1488113.595860] FS: 00007ea1da0a8700(0000) GS:ffff8a9affb80000(0000) knlGS:0000000000000000 [1488113.595861] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.595863] CR2: 00007efcd3da6000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.595863] Call Trace: [1488113.595865] [] queued_spin_lock_slowpath+0xb/0xf [1488113.595868] [] _raw_spin_lock+0x20/0x30 [1488113.595876] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.595886] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.595924] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.595962] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.595999] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.596037] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.596039] [] ? wake_up_state+0x20/0x20 [1488113.596077] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.596079] [] kthread+0xd1/0xe0 [1488113.596081] [] ? insert_kthread_work+0x40/0x40 [1488113.596082] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.596085] [] ? insert_kthread_work+0x40/0x40 [1488113.596104] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.667813] NMI watchdog: BUG: soft lockup - CPU#58 stuck for 22s! [ptlrpcd_01_07:54175] [1488113.667840] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.667847] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.667850] CPU: 58 PID: 54175 Comm: ptlrpcd_01_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.667851] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.667852] task: ffff8abaf4da9040 ti: ffff8abaee6dc000 task.ti: ffff8abaee6dc000 [1488113.667855] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.667856] RSP: 0018:ffff8abaee6dfb78 EFLAGS: 00000246 [1488113.667857] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001d10000 [1488113.667858] RDX: ffff8afb3f89b780 RSI: 0000000000b90001 RDI: ffff8adafa7b1b00 [1488113.667860] RBP: ffff8abaee6dfb78 R08: ffff8a9affbdb780 R09: 0000000000000000 [1488113.667860] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a950c268600 [1488113.667861] R13: 0005c4aa17a59da0 R14: ffff8a8c35a04e00 R15: 0000000000000007 [1488113.667863] FS: 00007ea1ba869700(0000) GS:ffff8a9affbc0000(0000) knlGS:0000000000000000 [1488113.667864] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.667865] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.667866] Call Trace: [1488113.667868] [] queued_spin_lock_slowpath+0xb/0xf [1488113.667870] [] _raw_spin_lock+0x20/0x30 [1488113.667878] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.667889] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.667927] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.667965] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.668002] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.668040] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.668043] [] ? wake_up_state+0x20/0x20 [1488113.668081] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.668083] [] kthread+0xd1/0xe0 [1488113.668085] [] ? insert_kthread_work+0x40/0x40 [1488113.668087] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.668089] [] ? insert_kthread_work+0x40/0x40 [1488113.668109] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.677813] NMI watchdog: BUG: soft lockup - CPU#59 stuck for 22s! [ptlrpcd_05_00:54216] [1488113.677839] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.677846] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.677849] CPU: 59 PID: 54216 Comm: ptlrpcd_05_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.677850] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.677851] task: ffff8abaee638000 ti: ffff8abaf8220000 task.ti: ffff8abaf8220000 [1488113.677854] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.677855] RSP: 0018:ffff8abaf8223b78 EFLAGS: 00000246 [1488113.677856] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001d90000 [1488113.677857] RDX: ffff8a9affb5b780 RSI: 0000000001510001 RDI: ffff8adafa7b1b00 [1488113.677857] RBP: ffff8abaf8223b78 R08: ffff8adaffbdb780 R09: 0000000000000000 [1488113.677858] R10: 0000000000000000 R11: 000000000000000f R12: ffff8acd69a63c00 [1488113.677860] R13: 0005c4aa178827a0 R14: ffff8ad241ea4200 R15: 0000000000000007 [1488113.677861] FS: 00007f8c263ff700(0000) GS:ffff8adaffbc0000(0000) knlGS:0000000000000000 [1488113.677862] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.677864] CR2: 000000000230d6a8 CR3: 00000057926fe000 CR4: 00000000003407e0 [1488113.677864] Call Trace: [1488113.677866] [] queued_spin_lock_slowpath+0xb/0xf [1488113.677869] [] _raw_spin_lock+0x20/0x30 [1488113.677876] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.677887] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.677925] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.677963] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.678000] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.678038] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.678041] [] ? wake_up_state+0x20/0x20 [1488113.678079] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.678081] [] kthread+0xd1/0xe0 [1488113.678083] [] ? insert_kthread_work+0x40/0x40 [1488113.678085] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.678087] [] ? insert_kthread_work+0x40/0x40 [1488113.678106] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.714816] NMI watchdog: BUG: soft lockup - CPU#63 stuck for 22s! [ptlrpcd_07_06:54246] [1488113.714845] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.714852] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.714855] CPU: 63 PID: 54246 Comm: ptlrpcd_07_06 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.714856] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.714857] task: ffff8abafc522080 ti: ffff8abaf4f54000 task.ti: ffff8abaf4f54000 [1488113.714860] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.714861] RSP: 0018:ffff8abaf4f57b78 EFLAGS: 00000246 [1488113.714862] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000001f90000 [1488113.714863] RDX: ffff8a9affa9b780 RSI: 0000000000910000 RDI: ffff8adafa7b1b00 [1488113.714864] RBP: ffff8abaf4f57b78 R08: ffff8afb3f9db780 R09: 0000000000000000 [1488113.714865] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aed569e2a00 [1488113.714866] R13: 0005c4aa17c51860 R14: ffff8aed24f64b00 R15: 0000000000000007 [1488113.714868] FS: 00007ea1d389b700(0000) GS:ffff8afb3f9c0000(0000) knlGS:0000000000000000 [1488113.714868] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.714870] CR2: 00007ecf3e8c7ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.714870] Call Trace: [1488113.714873] [] queued_spin_lock_slowpath+0xb/0xf [1488113.714875] [] _raw_spin_lock+0x20/0x30 [1488113.714883] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.714894] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.714932] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.714970] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.715007] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.715046] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.715048] [] ? wake_up_state+0x20/0x20 [1488113.715086] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.715087] [] kthread+0xd1/0xe0 [1488113.715089] [] ? insert_kthread_work+0x40/0x40 [1488113.715091] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.715093] [] ? insert_kthread_work+0x40/0x40 [1488113.715113] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.761819] NMI watchdog: BUG: soft lockup - CPU#68 stuck for 22s! [ptlrpcd_02_02:54182] [1488113.761847] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.761854] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.761857] CPU: 68 PID: 54182 Comm: ptlrpcd_02_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.761858] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.761859] task: ffff8abaee641040 ti: ffff8abafcdf0000 task.ti: ffff8abafcdf0000 [1488113.761862] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.761863] RSP: 0018:ffff8abafcdf3b78 EFLAGS: 00000246 [1488113.761864] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002210000 [1488113.761865] RDX: ffff8aaaffbdb780 RSI: 0000000001e10001 RDI: ffff8adafa7b1b00 [1488113.761866] RBP: ffff8abafcdf3b78 R08: ffff8aaaffc1b780 R09: 0000000000000000 [1488113.761867] R10: 0000000000000000 R11: 000000000000000f R12: ffff8aa4ef142800 [1488113.761868] R13: 0005c4aa17cce1a0 R14: ffff8aa4721f3900 R15: 0000000000000007 [1488113.761870] FS: 00007ea149112700(0000) GS:ffff8aaaffc00000(0000) knlGS:0000000000000000 [1488113.761871] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.761872] CR2: 00007f8eefd2e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.761873] Call Trace: [1488113.761875] [] queued_spin_lock_slowpath+0xb/0xf [1488113.761877] [] _raw_spin_lock+0x20/0x30 [1488113.761884] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.761895] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.761933] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.761970] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.762007] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.762045] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.762048] [] ? wake_up_state+0x20/0x20 [1488113.762085] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.762088] [] kthread+0xd1/0xe0 [1488113.762089] [] ? insert_kthread_work+0x40/0x40 [1488113.762091] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.762093] [] ? insert_kthread_work+0x40/0x40 [1488113.762113] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.904828] NMI watchdog: BUG: soft lockup - CPU#84 stuck for 23s! [ptlrpcd_02_00:54180] [1488113.904854] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.904861] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.904864] CPU: 84 PID: 54180 Comm: ptlrpcd_02_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.904865] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.904866] task: ffff8abaf4dae180 ti: ffff8abaecadc000 task.ti: ffff8abaecadc000 [1488113.904869] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.904870] RSP: 0018:ffff8abaecadfb78 EFLAGS: 00000246 [1488113.904871] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002a10000 [1488113.904872] RDX: ffff8aaaffb1b780 RSI: 0000000001210001 RDI: ffff8adafa7b1b00 [1488113.904873] RBP: ffff8abaecadfb78 R08: ffff8aaaffc9b780 R09: 0000000000000000 [1488113.904874] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9bffcd7200 [1488113.904875] R13: 0005c4aa17cd8670 R14: ffff8aa639e9aa00 R15: 0000000000000007 [1488113.904877] FS: 00007ea1c8885700(0000) GS:ffff8aaaffc80000(0000) knlGS:0000000000000000 [1488113.904878] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.904879] CR2: 00007ee6f34aa006 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.904880] Call Trace: [1488113.904882] [] queued_spin_lock_slowpath+0xb/0xf [1488113.904884] [] _raw_spin_lock+0x20/0x30 [1488113.904892] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.904903] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.904941] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.904978] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.905016] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.905054] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.905056] [] ? wake_up_state+0x20/0x20 [1488113.905095] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.905096] [] kthread+0xd1/0xe0 [1488113.905098] [] ? insert_kthread_work+0x40/0x40 [1488113.905100] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.905102] [] ? insert_kthread_work+0x40/0x40 [1488113.905122] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488113.996834] NMI watchdog: BUG: soft lockup - CPU#94 stuck for 23s! [ptlrpcd_03_02:54194] [1488113.996860] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488113.996868] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488113.996870] CPU: 94 PID: 54194 Comm: ptlrpcd_03_02 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488113.996871] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488113.996872] task: ffff8abaebafe180 ti: ffff8abaf8e10000 task.ti: ffff8abaf8e10000 [1488113.996876] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488113.996877] RSP: 0018:ffff8abaf8e13b78 EFLAGS: 00000246 [1488113.996878] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002f10000 [1488113.996878] RDX: ffff8a8aff2db780 RSI: 0000000000c10001 RDI: ffff8adafa7b1b00 [1488113.996879] RBP: ffff8abaf8e13b78 R08: ffff8abaffcdb780 R09: 0000000000000000 [1488113.996881] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab1562e0600 [1488113.996882] R13: 0005c4aa17cca680 R14: ffff8ab8fa384b00 R15: 0000000000000007 [1488113.996883] FS: 00007ea1d1897700(0000) GS:ffff8abaffcc0000(0000) knlGS:0000000000000000 [1488113.996884] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488113.996886] CR2: 00007f47fafb9014 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488113.996886] Call Trace: [1488113.996888] [] queued_spin_lock_slowpath+0xb/0xf [1488113.996891] [] _raw_spin_lock+0x20/0x30 [1488113.996899] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488113.996909] [] LNetMDUnlink+0x78/0x180 [lnet] [1488113.996948] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488113.996985] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488113.997022] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488113.997061] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488113.997063] [] ? wake_up_state+0x20/0x20 [1488113.997102] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488113.997104] [] kthread+0xd1/0xe0 [1488113.997106] [] ? insert_kthread_work+0x40/0x40 [1488113.997108] [] ret_from_fork_nospec_begin+0xe/0x21 [1488113.997109] [] ? insert_kthread_work+0x40/0x40 [1488113.997130] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488117.159043] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ptlrpcd_01_10:54178] [1488117.159070] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.159077] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.159080] CPU: 2 PID: 54178 Comm: ptlrpcd_01_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.159080] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.159081] task: ffff8abaf4dac100 ti: ffff8abafd6fc000 task.ti: ffff8abafd6fc000 [1488117.159085] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488117.159085] RSP: 0018:ffff8abafd6ffb78 EFLAGS: 00000246 [1488117.159086] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000110000 [1488117.159087] RDX: ffff8a9affc1b780 RSI: 0000000002110001 RDI: ffff8adafa7b1b00 [1488117.159089] RBP: ffff8abafd6ffb78 R08: ffff8a9affa1b780 R09: 0000000000000000 [1488117.159090] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a94eddae200 [1488117.159090] R13: 0005c4aa179f63f0 R14: ffff8a97eef74200 R15: 0000000000000007 [1488117.159092] FS: 00007ea1cb08a700(0000) GS:ffff8a9affa00000(0000) knlGS:0000000000000000 [1488117.159093] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.159094] CR2: 00007fd37fb66000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.159095] Call Trace: [1488117.159097] [] queued_spin_lock_slowpath+0xb/0xf [1488117.159099] [] _raw_spin_lock+0x20/0x30 [1488117.159107] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.159117] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.159155] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.159193] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.159229] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.159268] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.159270] [] ? wake_up_state+0x20/0x20 [1488117.159308] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.159310] [] kthread+0xd1/0xe0 [1488117.159312] [] ? insert_kthread_work+0x40/0x40 [1488117.159314] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.159316] [] ? insert_kthread_work+0x40/0x40 [1488117.159335] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488117.178043] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [ptlrpcd_02_10:54190] [1488117.178071] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.178078] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.178081] CPU: 4 PID: 54190 Comm: ptlrpcd_02_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.178082] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.178083] task: ffff8abaebafa080 ti: ffff8abaeca8c000 task.ti: ffff8abaeca8c000 [1488117.178086] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488117.178087] RSP: 0018:ffff8abaeca8fb78 EFLAGS: 00000246 [1488117.178088] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000210000 [1488117.178089] RDX: ffff8adaffc9b780 RSI: 0000000002990001 RDI: ffff8adafa7b1b00 [1488117.178090] RBP: ffff8abaeca8fb78 R08: ffff8aaaffa1b780 R09: 0000000000000000 [1488117.178091] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a8fde494e00 [1488117.178092] R13: 0005c4aa17cd4900 R14: ffff8aa3ff308000 R15: 0000000000000007 [1488117.178094] FS: 00007ea1d509e700(0000) GS:ffff8aaaffa00000(0000) knlGS:0000000000000000 [1488117.178095] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.178096] CR2: 0000000000453b20 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.178097] Call Trace: [1488117.178099] [] queued_spin_lock_slowpath+0xb/0xf [1488117.178101] [] _raw_spin_lock+0x20/0x30 [1488117.178109] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.178120] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.178158] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.178197] [] ? deuuidify+0x90/0x90 [ptlrpc] [1488117.178235] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.178272] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.178310] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.178313] [] ? wake_up_state+0x20/0x20 [1488117.178351] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.178353] [] kthread+0xd1/0xe0 [1488117.178355] [] ? insert_kthread_work+0x40/0x40 [1488117.178357] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.178359] [] ? insert_kthread_work+0x40/0x40 [1488117.178379] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488117.195045] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ptlrpcd_03_11:54203] [1488117.195072] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.195080] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.195082] CPU: 6 PID: 54203 Comm: ptlrpcd_03_11 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.195083] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.195084] task: ffff8aaba9651040 ti: ffff8abaed288000 task.ti: ffff8abaed288000 [1488117.195087] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488117.195088] RSP: 0018:ffff8abaed28bb78 EFLAGS: 00000246 [1488117.195089] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000310000 [1488117.195090] RDX: ffff8a9affadb780 RSI: 0000000000d10001 RDI: ffff8adafa7b1b00 [1488117.195091] RBP: ffff8abaed28bb78 R08: ffff8abaffa1b780 R09: 0000000000000000 [1488117.195092] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab36218f200 [1488117.195093] R13: 0005c4aa17d36130 R14: ffff8ab5a74fbc00 R15: 0000000000000007 [1488117.195095] FS: 00007ea1dc8ad700(0000) GS:ffff8abaffa00000(0000) knlGS:0000000000000000 [1488117.195096] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.195097] CR2: 00007f5ff48d9000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.195098] Call Trace: [1488117.195100] [] queued_spin_lock_slowpath+0xb/0xf [1488117.195102] [] _raw_spin_lock+0x20/0x30 [1488117.195110] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.195121] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.195159] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.195196] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.195233] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.195272] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.195274] [] ? wake_up_state+0x20/0x20 [1488117.195312] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.195314] [] kthread+0xd1/0xe0 [1488117.195316] [] ? insert_kthread_work+0x40/0x40 [1488117.195318] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.195319] [] ? insert_kthread_work+0x40/0x40 [1488117.195339] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488117.205046] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ptlrpcd_07_03:54243] [1488117.205072] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.205080] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.205083] CPU: 7 PID: 54243 Comm: ptlrpcd_07_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.205084] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.205085] task: ffff8abafbc7e180 ti: ffff8abafbb24000 task.ti: ffff8abafbb24000 [1488117.205088] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488117.205089] RSP: 0018:ffff8abafbb27b78 EFLAGS: 00000246 [1488117.205090] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000390000 [1488117.205091] RDX: ffff8acaffcdb780 RSI: 0000000002c90001 RDI: ffff8adafa7b1b00 [1488117.205092] RBP: ffff8abafbb27b78 R08: ffff8afb3f81b780 R09: 0000000000000000 [1488117.205093] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af1a1f2b200 [1488117.205094] R13: 0005c4aa17ccfca0 R14: ffff8af090a4ef00 R15: 0000000000000007 [1488117.205096] FS: 00007ea1ca889700(0000) GS:ffff8afb3f800000(0000) knlGS:0000000000000000 [1488117.205096] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.205097] CR2: 00007ee55cb4bff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.205098] Call Trace: [1488117.205101] [] queued_spin_lock_slowpath+0xb/0xf [1488117.205103] [] _raw_spin_lock+0x20/0x30 [1488117.205111] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.205121] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.205160] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.205198] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.205235] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.205274] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.205276] [] ? wake_up_state+0x20/0x20 [1488117.205314] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.205317] [] kthread+0xd1/0xe0 [1488117.205318] [] ? insert_kthread_work+0x40/0x40 [1488117.205320] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.205322] [] ? insert_kthread_work+0x40/0x40 [1488117.205342] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488117.223046] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ptlrpcd_04_07:54211] [1488117.223072] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.223079] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.223082] CPU: 9 PID: 54211 Comm: ptlrpcd_04_07 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.223083] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.223084] task: ffff8abaf82ca080 ti: ffff8abafd344000 task.ti: ffff8abafd344000 [1488117.223087] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488117.223088] RSP: 0018:ffff8abafd347b78 EFLAGS: 00000246 [1488117.223089] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000490000 [1488117.223090] RDX: ffff8acaffc5b780 RSI: 0000000002490001 RDI: ffff8adafa7b1b00 [1488117.223091] RBP: ffff8abafd347b78 R08: ffff8acaffa5b780 R09: 0000000000000000 [1488117.223092] R10: 0000000000000000 R11: 000000000000000f R12: ffff8abc1b613c00 [1488117.223093] R13: 0005c4aa17cc1560 R14: ffff8ac421352a00 R15: 0000000000000007 [1488117.223095] FS: 00007ea1b0054700(0000) GS:ffff8acaffa40000(0000) knlGS:0000000000000000 [1488117.223096] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.223097] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.223098] Call Trace: [1488117.223100] [] queued_spin_lock_slowpath+0xb/0xf [1488117.223102] [] _raw_spin_lock+0x20/0x30 [1488117.223110] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.223121] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.223159] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.223197] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.223234] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.223272] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.223275] [] ? wake_up_state+0x20/0x20 [1488117.223313] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.223315] [] kthread+0xd1/0xe0 [1488117.223317] [] ? insert_kthread_work+0x40/0x40 [1488117.223319] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.223321] [] ? insert_kthread_work+0x40/0x40 [1488117.223341] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488117.233048] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [ptlrpcd_01_05:54173] [1488117.233074] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.233082] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.233084] CPU: 10 PID: 54173 Comm: ptlrpcd_01_05 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.233085] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.233087] task: ffff8abaf6346180 ti: ffff8abaf3da8000 task.ti: ffff8abaf3da8000 [1488117.233090] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488117.233091] RSP: 0018:ffff8abaf3dabb78 EFLAGS: 00000246 [1488117.233092] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000510000 [1488117.233093] RDX: ffff8afb3fa1b780 RSI: 0000000002390001 RDI: ffff8adafa7b1b00 [1488117.233093] RBP: ffff8abaf3dabb78 R08: ffff8a9affa5b780 R09: 0000000000000000 [1488117.233095] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a90a36b8a00 [1488117.233096] R13: 0005c4aa17a6f150 R14: ffff8a90848c4200 R15: 0000000000000007 [1488117.233098] FS: 00007ea1b2859700(0000) GS:ffff8a9affa40000(0000) knlGS:0000000000000000 [1488117.233098] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.233100] CR2: 00007ee89ec1e000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.233100] Call Trace: [1488117.233102] [] queued_spin_lock_slowpath+0xb/0xf [1488117.233104] [] _raw_spin_lock+0x20/0x30 [1488117.233112] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.233123] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.233161] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.233199] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.233236] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.233275] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.233277] [] ? wake_up_state+0x20/0x20 [1488117.233315] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.233317] [] kthread+0xd1/0xe0 [1488117.233319] [] ? insert_kthread_work+0x40/0x40 [1488117.233321] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.233323] [] ? insert_kthread_work+0x40/0x40 [1488117.233343] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488117.250048] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ptlrpcd_02_03:54183] [1488117.250073] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.250081] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.250083] CPU: 12 PID: 54183 Comm: ptlrpcd_02_03 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.250084] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.250085] task: ffff8abaee642080 ti: ffff8abaf4528000 task.ti: ffff8abaf4528000 [1488117.250088] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488117.250089] RSP: 0018:ffff8abaf452bb78 EFLAGS: 00000246 [1488117.250090] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000610000 [1488117.250091] RDX: ffff8adaffb5b780 RSI: 0000000001590001 RDI: ffff8adafa7b1b00 [1488117.250092] RBP: ffff8abaf452bb78 R08: ffff8aaaffa5b780 R09: 0000000000000000 [1488117.250094] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a9f7c35a000 [1488117.250094] R13: 0005c4aa17cf54e0 R14: ffff8aa6838d2a00 R15: 0000000000000007 [1488117.250096] FS: 00007fc0d2114880(0000) GS:ffff8aaaffa40000(0000) knlGS:0000000000000000 [1488117.250097] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.250098] CR2: 00007fc0cc4ae000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.250099] Call Trace: [1488117.250101] [] queued_spin_lock_slowpath+0xb/0xf [1488117.250103] [] _raw_spin_lock+0x20/0x30 [1488117.250111] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.250121] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.250159] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.250198] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.250235] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.250273] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.250275] [] ? wake_up_state+0x20/0x20 [1488117.250314] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.250316] [] kthread+0xd1/0xe0 [1488117.250317] [] ? insert_kthread_work+0x40/0x40 [1488117.250319] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.250322] [] ? insert_kthread_work+0x40/0x40 [1488117.250341] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488117.269049] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [ptlrpcd_03_00:54192] [1488117.269076] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.269084] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.269086] CPU: 14 PID: 54192 Comm: ptlrpcd_03_00 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.269087] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.269088] task: ffff8abaebafc100 ti: ffff8abaecb70000 task.ti: ffff8abaecb70000 [1488117.269092] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488117.269093] RSP: 0018:ffff8abaecb73b78 EFLAGS: 00000246 [1488117.269094] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000710000 [1488117.269095] RDX: ffff8a9affc9b780 RSI: 0000000002910001 RDI: ffff8adafa7b1b00 [1488117.269096] RBP: ffff8abaecb73b78 R08: ffff8abaffa5b780 R09: 0000000000000000 [1488117.269097] R10: 0000000000000000 R11: 000000000000000f R12: ffff8ab181efec00 [1488117.269099] R13: 0005c4aa17cb8650 R14: ffff8ab98a532700 R15: 0000000000000007 [1488117.269100] FS: 00007ea1d309a700(0000) GS:ffff8abaffa40000(0000) knlGS:0000000000000000 [1488117.269101] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.269102] CR2: 00007ef9d5eb2334 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.269103] Call Trace: [1488117.269106] [] queued_spin_lock_slowpath+0xb/0xf [1488117.269108] [] _raw_spin_lock+0x20/0x30 [1488117.269115] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.269126] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.269164] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.269202] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.269239] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.269278] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.269280] [] ? wake_up_state+0x20/0x20 [1488117.269318] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.269321] [] kthread+0xd1/0xe0 [1488117.269322] [] ? insert_kthread_work+0x40/0x40 [1488117.269324] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.269326] [] ? insert_kthread_work+0x40/0x40 [1488117.269347] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488117.277050] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ptlrpcd_07_10:54250] [1488117.277078] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.277086] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.277088] CPU: 15 PID: 54250 Comm: ptlrpcd_07_10 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.277089] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.277091] task: ffff8abafc526180 ti: ffff8abafbe28000 task.ti: ffff8abafbe28000 [1488117.277094] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [1488117.277095] RSP: 0018:ffff8abafbe2bb78 EFLAGS: 00000246 [1488117.277096] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000000790000 [1488117.277097] RDX: ffff8aaaffc5b780 RSI: 0000000002610001 RDI: ffff8adafa7b1b00 [1488117.277097] RBP: ffff8abafbe2bb78 R08: ffff8afb3f85b780 R09: 0000000000000000 [1488117.277099] R10: 0000000000000000 R11: 000000000000000f R12: ffff8af5d3316800 [1488117.277100] R13: 0005c4aa17c32eb0 R14: ffff8af0e4b25400 R15: 0000000000000007 [1488117.277102] FS: 00007ea1b9066700(0000) GS:ffff8afb3f840000(0000) knlGS:0000000000000000 [1488117.277102] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.277104] CR2: 00007ecf3e917ff6 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.277104] Call Trace: [1488117.277107] [] queued_spin_lock_slowpath+0xb/0xf [1488117.277109] [] _raw_spin_lock+0x20/0x30 [1488117.277116] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.277127] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.277165] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.277203] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.277240] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.277279] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.277281] [] ? wake_up_state+0x20/0x20 [1488117.277319] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.277321] [] kthread+0xd1/0xe0 [1488117.277323] [] ? insert_kthread_work+0x40/0x40 [1488117.277325] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.277326] [] ? insert_kthread_work+0x40/0x40 [1488117.277347] Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [1488117.289950] NMI watchdog: Watchdog detected hard LOCKUP on cpu 72 [1488117.289977] Modules linked in: mgc(OE) lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mpt2sas raid_class scsi_transport_sas mptctl mptbase rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul sg ghash_clmulni_intel dm_multipath dm_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si ipmi_devintf pcspkr k10temp i2c_piix4 ccp ipmi_msghandler acpi_power_meter ip_tables ext4 mlx5_ib(OE) mbcache ib_uverbs(OE) jbd2 ib_core(OE) sd_mod crc_t10dif crct10dif_generic mlx5_core(OE) [1488117.289984] i2c_algo_bit drm_kms_helper mlxfw(OE) syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm devlink ahci crct10dif_common crc32c_intel drm megaraid_sas mlx_compat(OE) libahci libata drm_panel_orientation_quirks tg3 ptp pps_core [last unloaded: libcfs] [1488117.289987] CPU: 72 PID: 54160 Comm: ptlrpcd_00_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.289988] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.289989] task: ffff8abaf8158000 ti: ffff8abaf4a70000 task.ti: ffff8abaf4a70000 [1488117.289993] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [1488117.289994] RSP: 0018:ffff8abaf4a73b78 EFLAGS: 00000246 [1488117.289995] RAX: 0000000000000000 RBX: ffffffffc0ff5a97 RCX: 0000000002410000 [1488117.289996] RDX: ffff8aeaffadb780 RSI: 0000000000e90001 RDI: ffff8adafa7b1b00 [1488117.289996] RBP: ffff8abaf4a73b78 R08: ffff8a8aff45b780 R09: 0000000000000000 [1488117.289998] R10: 0000000000000000 R11: 000000000000000f R12: ffff8a98a92c0200 [1488117.289999] R13: 0005c4aa1879fac0 R14: ffff8a7b02ec6f00 R15: 0000000000000007 [1488117.290000] FS: 00007ea1ce090700(0000) GS:ffff8a8aff440000(0000) knlGS:0000000000000000 [1488117.290001] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1488117.290003] CR2: 00007fbad3d83000 CR3: 000000276ae10000 CR4: 00000000003407e0 [1488117.290003] Call Trace: [1488117.290006] [] queued_spin_lock_slowpath+0xb/0xf [1488117.290008] [] _raw_spin_lock+0x20/0x30 [1488117.290016] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.290027] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.290065] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.290102] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.290140] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.290178] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.290181] [] ? wake_up_state+0x20/0x20 [1488117.290219] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.290221] [] kthread+0xd1/0xe0 [1488117.290223] [] ? insert_kthread_work+0x40/0x40 [1488117.290225] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.290226] [] ? insert_kthread_work+0x40/0x40 [1488117.290247] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 60 b9 34 8a 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [1488117.290249] Kernel panic - not syncing: Hard LOCKUP [1488117.290251] CPU: 72 PID: 54160 Comm: ptlrpcd_00_04 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1488117.290252] Hardware name: Dell Inc. PowerEdge R7425/02MJ3T, BIOS 1.3.6 04/20/2018 [1488117.290252] Call Trace: [1488117.290256] [] dump_stack+0x19/0x1b [1488117.290258] [] panic+0xe8/0x21f [1488117.290261] [] nmi_panic+0x3f/0x40 [1488117.290265] [] watchdog_overflow_callback+0x121/0x140 [1488117.290269] [] __perf_event_overflow+0x57/0x100 [1488117.290272] [] perf_event_overflow+0x14/0x20 [1488117.290275] [] x86_pmu_handle_irq+0x140/0x1a0 [1488117.290279] [] ? ioremap_page_range+0x2e8/0x480 [1488117.290283] [] ? vunmap_page_range+0x234/0x470 [1488117.290285] [] ? unmap_kernel_range_noflush+0x11/0x20 [1488117.290289] [] ? ghes_copy_tofrom_phys+0x10e/0x210 [1488117.290291] [] ? ghes_read_estatus+0xa0/0x190 [1488117.290294] [] perf_event_nmi_handler+0x31/0x50 [1488117.290296] [] nmi_handle.isra.0+0x8c/0x150 [1488117.290298] [] do_nmi+0x218/0x460 [1488117.290300] [] end_repeat_nmi+0x1e/0x81 [1488117.290303] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488117.290305] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488117.290308] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488117.290310] [] queued_spin_lock_slowpath+0xb/0xf [1488117.290313] [] _raw_spin_lock_irqsave+0x37/0x40 [1488117.290315] [] rcu_dump_cpu_stacks+0x4b/0xd0 [1488117.290316] [] rcu_check_callbacks+0x442/0x730 [1488117.290319] [] ? tick_sched_do_timer+0x50/0x50 [1488117.290320] [] update_process_times+0x46/0x80 [1488117.290322] [] tick_sched_handle+0x30/0x70 [1488117.290324] [] tick_sched_timer+0x39/0x80 [1488117.290326] [] __hrtimer_run_queues+0xf3/0x270 [1488117.290328] [] hrtimer_interrupt+0xaf/0x1d0 [1488117.290331] [] local_apic_timer_interrupt+0x3b/0x60 [1488117.290332] [] smp_apic_timer_interrupt+0x43/0x60 [1488117.290335] [] apic_timer_interrupt+0x162/0x170 [1488117.290338] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [1488117.290340] [] queued_spin_lock_slowpath+0xb/0xf [1488117.290342] [] _raw_spin_lock+0x20/0x30 [1488117.290350] [] cfs_percpt_lock+0x58/0x110 [libcfs] [1488117.290360] [] LNetMDUnlink+0x78/0x180 [lnet] [1488117.290398] [] ptlrpc_unregister_reply+0xbf/0x790 [ptlrpc] [1488117.290436] [] ptlrpc_expire_one_request+0xee/0x520 [ptlrpc] [1488117.290473] [] ptlrpc_expired_set+0xaf/0x1a0 [ptlrpc] [1488117.290512] [] ptlrpcd+0x28c/0x550 [ptlrpc] [1488117.290514] [] ? wake_up_state+0x20/0x20 [1488117.290553] [] ? ptlrpcd_check+0x590/0x590 [ptlrpc] [1488117.290554] [] kthread+0xd1/0xe0 [1488117.290556] [] ? insert_kthread_work+0x40/0x40 [1488117.290558] [] ret_from_fork_nospec_begin+0xe/0x21 [1488117.290560] [] ? insert_kthread_work+0x40/0x40