[2642432.296320] CPU: 5 PID: 24592 Comm: ll_ost01_047 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.296321] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.296322] task: ffff9bff19d520e0 ti: ffff9c14566e8000 task.ti: ffff9c14566e8000 [2642432.296326] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.296327] RSP: 0018:ffff9c14566ebb68 EFLAGS: 00000246 [2642432.296327] RAX: 0000000000000000 RBX: ffff9c14566ebb18 RCX: 0000000000290000 [2642432.296328] RDX: ffff9c2d9d8db880 RSI: 0000000000b90101 RDI: ffff9c2a4a5112dc [2642432.296328] RBP: ffff9c14566ebb68 R08: ffff9c2d9d69b880 R09: 0000000000000000 [2642432.296329] R10: ffff9c147036d800 R11: ffff9c144aace780 R12: ffff9c1804c86050 [2642432.296329] R13: ffff9c2d53cbcc00 R14: 0000000084d60982 R15: 0000000000000000 [2642432.296330] FS: 0000000000000000(0000) GS:ffff9c2d9d680000(0000) knlGS:0000000000000000 [2642432.296331] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.296332] CR2: 00007fd092109140 CR3: 000000190da10000 CR4: 00000000007607e0 [2642432.296332] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.296333] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.296333] PKRU: 00000000 [2642432.296334] Call Trace: [2642432.296337] [] queued_spin_lock_slowpath+0xb/0xf [2642432.296340] [] _raw_spin_lock+0x20/0x30 [2642432.296362] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.296376] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.296398] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.296416] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.296436] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.296462] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.296486] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.296509] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.296513] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.296534] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.296553] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.296556] [] ? __wake_up+0x13/0x20 [2642432.296575] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.296594] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.296596] [] kthread+0xd1/0xe0 [2642432.296598] [] ? insert_kthread_work+0x40/0x40 [2642432.296599] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.296601] [] ? insert_kthread_work+0x40/0x40 [2642432.296615] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.301256] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ll_ost00_080:24753] [2642432.301279] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.301301] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.301303] CPU: 6 PID: 24753 Comm: ll_ost00_080 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.301304] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.301305] task: ffff9c143921c1c0 ti: ffff9c1439468000 task.ti: ffff9c1439468000 [2642432.301308] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.301308] RSP: 0018:ffff9c143946bb68 EFLAGS: 00000246 [2642432.301309] RAX: 0000000000000000 RBX: ffff9c143946bb18 RCX: 0000000000310000 [2642432.301310] RDX: ffff9c2d9d65b880 RSI: 0000000000190101 RDI: ffff9c2a4a5112dc [2642432.301310] RBP: ffff9c143946bb68 R08: ffff9c159f2db880 R09: 0000000000000000 [2642432.301311] R10: ffff9c147036d800 R11: ffff9c144a9eca40 R12: ffff9c1466a21850 [2642432.301311] R13: ffff9c1eec7fcc00 R14: 000000002fbe7713 R15: 0000000000000000 [2642432.301313] FS: 0000000000000000(0000) GS:ffff9c159f2c0000(0000) knlGS:0000000000000000 [2642432.301313] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.301314] CR2: 00007efcc5c24bc0 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.301315] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.301315] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.301316] PKRU: 00000000 [2642432.301316] Call Trace: [2642432.301319] [] queued_spin_lock_slowpath+0xb/0xf [2642432.301321] [] _raw_spin_lock+0x20/0x30 [2642432.301339] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.301354] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.301375] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.301393] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.301414] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.301439] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.301464] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.301487] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.301491] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.301512] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.301532] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.301534] [] ? wake_up_state+0x20/0x20 [2642432.301554] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.301574] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.301576] [] kthread+0xd1/0xe0 [2642432.301578] [] ? insert_kthread_work+0x40/0x40 [2642432.301579] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.301581] [] ? insert_kthread_work+0x40/0x40 [2642432.301596] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.305246] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ll_ost01_115:24785] [2642432.305268] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.305287] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.305289] CPU: 7 PID: 24785 Comm: ll_ost01_115 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.305290] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.305291] task: ffff9c143d551070 ti: ffff9c143d98c000 task.ti: ffff9c143d98c000 [2642432.305294] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2642432.305295] RSP: 0018:ffff9c143d98fb68 EFLAGS: 00000246 [2642432.305295] RAX: 0000000000000000 RBX: ffff9c143d98fb18 RCX: 0000000000390000 [2642432.305296] RDX: ffff9c159f25b880 RSI: 0000000000110101 RDI: ffff9c2a4a5112dc [2642432.305297] RBP: ffff9c143d98fb68 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2642432.305297] R10: ffff9c147036d800 R11: ffff9c14498ad840 R12: ffff9c1f3ae42050 [2642432.305298] R13: ffff9c29d22b3000 R14: 00000000ed379bf3 R15: 0000000000000000 [2642432.305299] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2642432.305299] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.305300] CR2: 00007fcb82c08000 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.305301] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.305301] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.305302] PKRU: 00000000 [2642432.305302] Call Trace: [2642432.305305] [] queued_spin_lock_slowpath+0xb/0xf [2642432.305307] [] _raw_spin_lock+0x20/0x30 [2642432.305322] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.305337] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.305358] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.305375] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.305395] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.305420] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.305445] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.305467] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.305471] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.305491] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.305510] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.305512] [] ? __wake_up+0x13/0x20 [2642432.305532] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.305552] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.305553] [] kthread+0xd1/0xe0 [2642432.305555] [] ? insert_kthread_work+0x40/0x40 [2642432.305556] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.305558] [] ? insert_kthread_work+0x40/0x40 [2642432.305572] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2642432.309239] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_cn00_044:57948] [2642432.309262] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.309282] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.309284] CPU: 8 PID: 57948 Comm: ldlm_cn00_044 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.309284] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.309285] task: ffff9c153ef8a0e0 ti: ffff9c10d02ec000 task.ti: ffff9c10d02ec000 [2642432.309306] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x224/0x810 [ptlrpc] [2642432.309306] RSP: 0018:ffff9c10d02efac0 EFLAGS: 00000287 [2642432.309307] RAX: ffff9c2b88b6dc40 RBX: ffff9c10d02efa70 RCX: 000000000000001f [2642432.309308] RDX: ffff9c07b829bd20 RSI: 000000000480a000 RDI: ffff9c2bdfd04c00 [2642432.309308] RBP: ffff9c10d02efb48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2642432.309309] R10: 000000000480a000 R11: 000000000480a000 R12: ffff9bff67c23000 [2642432.309309] R13: ffff9c10d02efb48 R14: 0000000000000004 R15: ffff9bff67c23000 [2642432.309310] FS: 0000000000000000(0000) GS:ffff9c159f300000(0000) knlGS:0000000000000000 [2642432.309311] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.309312] CR2: 00007f7d1a3d2bc0 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.309312] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.309313] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.309313] PKRU: 00000000 [2642432.309314] Call Trace: [2642432.309331] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2642432.309347] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2642432.309362] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2642432.309368] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2642432.309383] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2642432.309397] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2642432.309414] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2642432.309431] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2642432.309447] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2642432.309467] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.309487] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.309490] [] ? __wake_up+0x13/0x20 [2642432.309509] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.309529] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.309530] [] kthread+0xd1/0xe0 [2642432.309532] [] ? insert_kthread_work+0x40/0x40 [2642432.309534] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.309535] [] ? insert_kthread_work+0x40/0x40 [2642432.309549] Code: 39 d0 48 8d 42 a0 0f 84 b8 04 00 00 48 8b 75 b8 31 c9 eb 24 0f 1f 00 49 8b bf b8 00 00 00 48 39 b8 b8 00 00 00 74 2f 48 8b 50 60 <49> 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 [2642432.313226] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ll_ost01_110:24776] [2642432.313249] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.313268] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.313270] CPU: 9 PID: 24776 Comm: ll_ost01_110 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.313270] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.313271] task: ffff9c143ae062a0 ti: ffff9c143beb0000 task.ti: ffff9c143beb0000 [2642432.313274] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.313275] RSP: 0018:ffff9c143beb3b68 EFLAGS: 00000246 [2642432.313275] RAX: 0000000000000000 RBX: ffff9c143beb3b18 RCX: 0000000000490000 [2642432.313276] RDX: ffff9c159f4db880 RSI: 0000000000b10101 RDI: ffff9c2a4a5112dc [2642432.313276] RBP: ffff9c143beb3b68 R08: ffff9c2d9d71b880 R09: 0000000000000000 [2642432.313277] R10: ffff9c147036d800 R11: ffff9c144c6a9840 R12: ffff9c1f3ae41850 [2642432.313277] R13: ffff9c220c200000 R14: 0000000086f4995e R15: 0000000000000000 [2642432.313278] FS: 0000000000000000(0000) GS:ffff9c2d9d700000(0000) knlGS:0000000000000000 [2642432.313279] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.313280] CR2: 00000000006dee20 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.313280] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.313281] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.313281] PKRU: 00000000 [2642432.313282] Call Trace: [2642432.313284] [] queued_spin_lock_slowpath+0xb/0xf [2642432.313286] [] _raw_spin_lock+0x20/0x30 [2642432.313301] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.313316] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.313337] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.313354] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.313374] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.313400] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.313424] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.313446] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.313450] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.313470] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.313490] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.313492] [] ? __wake_up+0x13/0x20 [2642432.313512] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.313532] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.313533] [] kthread+0xd1/0xe0 [2642432.313535] [] ? insert_kthread_work+0x40/0x40 [2642432.313537] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.313538] [] ? insert_kthread_work+0x40/0x40 [2642432.313552] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.317217] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ll_ost00_029:24590] [2642432.317240] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.317260] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.317262] CPU: 10 PID: 24590 Comm: ll_ost00_029 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.317262] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.317263] task: ffff9c1564a8e2a0 ti: ffff9c14605e0000 task.ti: ffff9c14605e0000 [2642432.317266] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.317267] RSP: 0018:ffff9c14605e3b68 EFLAGS: 00000246 [2642432.317267] RAX: 0000000000000000 RBX: ffff9c14605e3b18 RCX: 0000000000510000 [2642432.317268] RDX: ffff9c2d9d89b880 RSI: 0000000000a90101 RDI: ffff9c2a4a5112dc [2642432.317268] RBP: ffff9c14605e3b68 R08: ffff9c159f35b880 R09: 0000000000000000 [2642432.317269] R10: ffff9c147036d800 R11: ffff9c144c6dc8c0 R12: ffff9c14670e3050 [2642432.317270] R13: ffff9c146de3e800 R14: 00000000bfe4d8b5 R15: 0000000000000000 [2642432.317271] FS: 0000000000000000(0000) GS:ffff9c159f340000(0000) knlGS:0000000000000000 [2642432.317271] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.317272] CR2: 00007fd0986b7c60 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.317273] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.317273] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.317273] PKRU: 00000000 [2642432.317274] Call Trace: [2642432.317277] [] queued_spin_lock_slowpath+0xb/0xf [2642432.317279] [] _raw_spin_lock+0x20/0x30 [2642432.317296] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.317311] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.317331] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.317349] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.317369] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.317394] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.317418] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.317441] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.317445] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.317465] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.317485] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.317487] [] ? __wake_up+0x13/0x20 [2642432.317507] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.317526] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.317528] [] kthread+0xd1/0xe0 [2642432.317530] [] ? insert_kthread_work+0x40/0x40 [2642432.317531] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.317533] [] ? insert_kthread_work+0x40/0x40 [2642432.317547] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.321207] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ll_ost01_099:24759] [2642432.321230] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.321249] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.321251] CPU: 11 PID: 24759 Comm: ll_ost01_099 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.321252] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.321252] task: ffff9c1439b43150 ti: ffff9c143b3ac000 task.ti: ffff9c143b3ac000 [2642432.321255] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.321256] RSP: 0018:ffff9c143b3afb68 EFLAGS: 00000246 [2642432.321257] RAX: 0000000000000000 RBX: ffff9c143b3afb18 RCX: 0000000000590000 [2642432.321257] RDX: ffff9c2d9d7db880 RSI: 0000000000790101 RDI: ffff9c2a4a5112dc [2642432.321258] RBP: ffff9c143b3afb68 R08: ffff9c2d9d75b880 R09: 0000000000000000 [2642432.321258] R10: ffff9c147036d800 R11: ffff9c144c6a99c0 R12: ffff9c1cb80a8050 [2642432.321259] R13: ffff9c1ba82bb000 R14: 0000000013818676 R15: 0000000000000000 [2642432.321260] FS: 0000000000000000(0000) GS:ffff9c2d9d740000(0000) knlGS:0000000000000000 [2642432.321260] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.321261] CR2: 000000c0013eb000 CR3: 000000190da10000 CR4: 00000000007607e0 [2642432.321262] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.321262] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.321263] PKRU: 00000000 [2642432.321263] Call Trace: [2642432.321266] [] queued_spin_lock_slowpath+0xb/0xf [2642432.321268] [] _raw_spin_lock+0x20/0x30 [2642432.321285] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.321300] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.321320] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.321338] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.321358] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.321383] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.321408] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.321430] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.321434] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.321455] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.321474] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.321476] [] ? __wake_up+0x13/0x20 [2642432.321496] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.321516] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.321517] [] kthread+0xd1/0xe0 [2642432.321519] [] ? insert_kthread_work+0x40/0x40 [2642432.321521] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.321522] [] ? insert_kthread_work+0x40/0x40 [2642432.321536] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.325198] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ll_ost00_079:24752] [2642432.325220] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.325240] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.325242] CPU: 12 PID: 24752 Comm: ll_ost00_079 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.325242] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.325243] task: ffff9c143921b150 ti: ffff9c14397c8000 task.ti: ffff9c14397c8000 [2642432.325246] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.325247] RSP: 0018:ffff9c14397cbb68 EFLAGS: 00000246 [2642432.325248] RAX: 0000000000000000 RBX: ffff9c14397cbb18 RCX: 0000000000610000 [2642432.325248] RDX: ffff9c2d9d79b880 RSI: 0000000000690101 RDI: ffff9c2a4a5112dc [2642432.325249] RBP: ffff9c14397cbb68 R08: ffff9c159f39b880 R09: 0000000000000000 [2642432.325249] R10: ffff9c147036d800 R11: ffff9c144a9ecb00 R12: ffff9c144ee48050 [2642432.325250] R13: ffff9c0863c05400 R14: 000000004138db45 R15: 0000000000000000 [2642432.325251] FS: 0000000000000000(0000) GS:ffff9c159f380000(0000) knlGS:0000000000000000 [2642432.325252] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.325252] CR2: 00000000006dee20 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.325253] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.325253] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.325254] PKRU: 00000000 [2642432.325254] Call Trace: [2642432.325257] [] queued_spin_lock_slowpath+0xb/0xf [2642432.325259] [] _raw_spin_lock+0x20/0x30 [2642432.325277] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.325292] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.325313] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.325331] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.325351] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.325376] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.325400] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.325423] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.325427] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.325447] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.325467] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.325469] [] ? wake_up_state+0x20/0x20 [2642432.325489] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.325509] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.325510] [] kthread+0xd1/0xe0 [2642432.325512] [] ? insert_kthread_work+0x40/0x40 [2642432.325513] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.325515] [] ? insert_kthread_work+0x40/0x40 [2642432.325529] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.329188] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ll_ost01_018:23726] [2642432.329210] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.329229] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.329231] CPU: 13 PID: 23726 Comm: ll_ost01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.329232] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.329232] task: ffff9c148e919070 ti: ffff9c143347c000 task.ti: ffff9c143347c000 [2642432.329235] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.329236] RSP: 0018:ffff9c143347fad0 EFLAGS: 00000246 [2642432.329237] RAX: 0000000000000000 RBX: ffff9c1f3ae43050 RCX: 0000000000690000 [2642432.329237] RDX: ffff9c159f35b880 RSI: 0000000000510101 RDI: ffff9c2a4a5112dc [2642432.329238] RBP: ffff9c143347fad0 R08: ffff9c2d9d79b880 R09: 0000000000000000 [2642432.329238] R10: ffff9bff3fc07500 R11: ffff9c2d7e734f40 R12: ffffffffc14b9237 [2642432.329239] R13: ffff9c143347fa58 R14: 0000000000000003 R15: 0000000000000003 [2642432.329240] FS: 0000000000000000(0000) GS:ffff9c2d9d780000(0000) knlGS:0000000000000000 [2642432.329240] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.329241] CR2: 00007fc5c9799bc0 CR3: 000000190da10000 CR4: 00000000007607e0 [2642432.329242] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.329242] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.329243] PKRU: 00000000 [2642432.329243] Call Trace: [2642432.329246] [] queued_spin_lock_slowpath+0xb/0xf [2642432.329248] [] _raw_spin_lock+0x20/0x30 [2642432.329260] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2642432.329277] [] ? ldlm_extent_alloc_lock+0x6c/0x280 [ptlrpc] [2642432.329292] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2642432.329298] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2642432.329303] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2642432.329320] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.329340] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.329366] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.329390] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.329412] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.329416] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.329436] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.329456] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.329458] [] ? __wake_up+0x13/0x20 [2642432.329477] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.329497] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.329499] [] kthread+0xd1/0xe0 [2642432.329500] [] ? insert_kthread_work+0x40/0x40 [2642432.329502] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.329503] [] ? insert_kthread_work+0x40/0x40 [2642432.329517] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.333178] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [ll_ost00_031:24594] [2642432.333201] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.333221] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.333223] CPU: 14 PID: 24594 Comm: ll_ost00_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.333223] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.333224] task: ffff9c1597ef1070 ti: ffff9c14542b0000 task.ti: ffff9c14542b0000 [2642432.333227] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2642432.333228] RSP: 0018:ffff9c14542b3b68 EFLAGS: 00000246 [2642432.333228] RAX: 0000000000000000 RBX: ffff9c14542b3b18 RCX: 0000000000710000 [2642432.333229] RDX: ffff9c159f49b880 RSI: 0000000000a10101 RDI: ffff9c2a4a5112dc [2642432.333230] RBP: ffff9c14542b3b68 R08: ffff9c159f3db880 R09: 0000000000000000 [2642432.333230] R10: ffff9c147036d800 R11: ffff9c144c6dcbc0 R12: ffff9c144ee4b850 [2642432.333231] R13: ffff9c25d893cc00 R14: 00000000c5f0618a R15: 0000000000000000 [2642432.333232] FS: 0000000000000000(0000) GS:ffff9c159f3c0000(0000) knlGS:0000000000000000 [2642432.333233] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.333234] CR2: 00007f9d0f438000 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.333234] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.333235] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.333235] PKRU: 00000000 [2642432.333236] Call Trace: [2642432.333239] [] queued_spin_lock_slowpath+0xb/0xf [2642432.333241] [] _raw_spin_lock+0x20/0x30 [2642432.333258] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.333273] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.333294] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.333311] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.333332] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.333357] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.333381] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.333404] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.333408] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.333428] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.333449] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.333451] [] ? __wake_up+0x13/0x20 [2642432.333471] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.333490] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.333492] [] kthread+0xd1/0xe0 [2642432.333494] [] ? insert_kthread_work+0x40/0x40 [2642432.333495] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.333496] [] ? insert_kthread_work+0x40/0x40 [2642432.333511] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2642432.338166] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ll_ost01_125:24806] [2642432.338188] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.338207] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.338209] CPU: 15 PID: 24806 Comm: ll_ost01_125 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.338209] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.338210] task: ffff9c144d351070 ti: ffff9c144d030000 task.ti: ffff9c144d030000 [2642432.338213] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.338214] RSP: 0018:ffff9c144d033b68 EFLAGS: 00000246 [2642432.338214] RAX: 0000000000000000 RBX: ffff9c144d033b18 RCX: 0000000000790000 [2642432.338215] RDX: ffff9c159f39b880 RSI: 0000000000610101 RDI: ffff9c2a4a5112dc [2642432.338215] RBP: ffff9c144d033b68 R08: ffff9c2d9d7db880 R09: 0000000000000000 [2642432.338216] R10: ffff9c147036d800 R11: ffff9c144a5173c0 R12: ffff9c2c7c54a050 [2642432.338216] R13: ffff9c280b6ae800 R14: 000000001a0aea37 R15: 0000000000000000 [2642432.338217] FS: 0000000000000000(0000) GS:ffff9c2d9d7c0000(0000) knlGS:0000000000000000 [2642432.338218] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.338219] CR2: 000000c00226d000 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.338219] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.338220] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.338220] PKRU: 00000000 [2642432.338220] Call Trace: [2642432.338223] [] queued_spin_lock_slowpath+0xb/0xf [2642432.338225] [] _raw_spin_lock+0x20/0x30 [2642432.338241] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.338255] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.338276] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.338294] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.338314] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.338339] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.338363] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.338385] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.338389] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.338409] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.338429] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.338431] [] ? __wake_up+0x13/0x20 [2642432.338450] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.338470] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.338471] [] kthread+0xd1/0xe0 [2642432.338473] [] ? insert_kthread_work+0x40/0x40 [2642432.338474] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.338476] [] ? insert_kthread_work+0x40/0x40 [2642432.338490] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.342156] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ll_ost00_004:22873] [2642432.342178] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.342198] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.342200] CPU: 16 PID: 22873 Comm: ll_ost00_004 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.342200] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.342201] task: ffff9c2c8bbb41c0 ti: ffff9c2c8bb20000 task.ti: ffff9c2c8bb20000 [2642432.342204] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.342205] RSP: 0018:ffff9c2c8bb23b68 EFLAGS: 00000246 [2642432.342206] RAX: 0000000000000000 RBX: ffff9c2c8bb23b18 RCX: 0000000000810000 [2642432.342206] RDX: ffff9c2d9d85b880 RSI: 0000000000990101 RDI: ffff9c2a4a5112dc [2642432.342207] RBP: ffff9c2c8bb23b68 R08: ffff9c159f41b880 R09: 0000000000000000 [2642432.342207] R10: ffff9c147036d800 R11: ffff9c157630fcc0 R12: ffff9c13195cc050 [2642432.342208] R13: ffff9c2b3d0a0400 R14: 00000000bf4764f5 R15: 0000000000000000 [2642432.342209] FS: 0000000000000000(0000) GS:ffff9c159f400000(0000) knlGS:0000000000000000 [2642432.342210] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.342210] CR2: 00007fcb806f1e70 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.342211] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.342211] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.342212] PKRU: 00000000 [2642432.342212] Call Trace: [2642432.342215] [] queued_spin_lock_slowpath+0xb/0xf [2642432.342217] [] _raw_spin_lock+0x20/0x30 [2642432.342233] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.342248] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.342269] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.342287] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.342307] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.342332] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.342357] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.342380] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.342384] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.342405] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.342424] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.342427] [] ? __wake_up+0x13/0x20 [2642432.342446] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.342466] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.342468] [] kthread+0xd1/0xe0 [2642432.342469] [] ? insert_kthread_work+0x40/0x40 [2642432.342471] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.342472] [] ? insert_kthread_work+0x40/0x40 [2642432.342487] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.346146] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ll_ost01_138:25266] [2642432.346168] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.346187] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.346189] CPU: 17 PID: 25266 Comm: ll_ost01_138 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.346189] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.346190] task: ffff9c2d9bb75230 ti: ffff9c2c642a0000 task.ti: ffff9c2c642a0000 [2642432.346193] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.346194] RSP: 0018:ffff9c2c642a3b68 EFLAGS: 00000246 [2642432.346194] RAX: 0000000000000000 RBX: ffff9c2c642a3b18 RCX: 0000000000890000 [2642432.346195] RDX: ffff9c159f45b880 RSI: 0000000000910101 RDI: ffff9c2a4a5112dc [2642432.346195] RBP: ffff9c2c642a3b68 R08: ffff9c2d9d81b880 R09: 0000000000000000 [2642432.346196] R10: ffff9c147036d800 R11: ffff9c1486cfcc40 R12: ffff9c1f3ae45050 [2642432.346197] R13: ffff9c20ee675800 R14: 00000000f554872d R15: 0000000000000000 [2642432.346198] FS: 0000000000000000(0000) GS:ffff9c2d9d800000(0000) knlGS:0000000000000000 [2642432.346198] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.346199] CR2: 00007f5f28a66bc0 CR3: 000000190da10000 CR4: 00000000007607e0 [2642432.346200] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.346200] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.346200] PKRU: 00000000 [2642432.346201] Call Trace: [2642432.346204] [] queued_spin_lock_slowpath+0xb/0xf [2642432.346206] [] _raw_spin_lock+0x20/0x30 [2642432.346221] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.346235] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.346256] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.346274] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.346294] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.346319] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.346343] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.346365] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.346369] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.346390] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.346410] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.346412] [] ? __wake_up+0x13/0x20 [2642432.346431] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.346451] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.346453] [] kthread+0xd1/0xe0 [2642432.346454] [] ? insert_kthread_work+0x40/0x40 [2642432.346456] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.346457] [] ? insert_kthread_work+0x40/0x40 [2642432.346471] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.350138] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ll_ost00_106:24816] [2642432.350159] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.350179] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.350182] CPU: 18 PID: 24816 Comm: ll_ost00_106 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.350182] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.350183] task: ffff9c144cd641c0 ti: ffff9c144c874000 task.ti: ffff9c144c874000 [2642432.350186] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2642432.350187] RSP: 0018:ffff9c144c877b68 EFLAGS: 00000202 [2642432.350187] RAX: 0000000000000101 RBX: ffff9c144c877b18 RCX: 0000000000910000 [2642432.350188] RDX: 0000000000390101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2642432.350188] RBP: ffff9c144c877b68 R08: ffff9c159f45b880 R09: 0000000000000000 [2642432.350189] R10: ffff9c147036d800 R11: ffff9c144a9e8b00 R12: ffff9c144ee49050 [2642432.350189] R13: ffff9c146addac00 R14: 00000000f65419ee R15: 0000000000000000 [2642432.350191] FS: 0000000000000000(0000) GS:ffff9c159f440000(0000) knlGS:0000000000000000 [2642432.350191] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.350192] CR2: 000000c001961010 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.350192] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.350193] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.350193] PKRU: 00000000 [2642432.350194] Call Trace: [2642432.350197] [] queued_spin_lock_slowpath+0xb/0xf [2642432.350198] [] _raw_spin_lock+0x20/0x30 [2642432.350215] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.350230] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.350251] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.350268] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.350289] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.350314] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.350338] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.350361] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.350365] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.350385] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.350405] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.350407] [] ? __wake_up+0x13/0x20 [2642432.350426] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.350446] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.350448] [] kthread+0xd1/0xe0 [2642432.350449] [] ? insert_kthread_work+0x40/0x40 [2642432.350451] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.350452] [] ? insert_kthread_work+0x40/0x40 [2642432.350466] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2642432.354127] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ll_ost01_006:22881] [2642432.354148] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.354168] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.354169] CPU: 19 PID: 22881 Comm: ll_ost01_006 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.354170] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.354171] task: ffff9c2d99c05230 ti: ffff9c2c8bbc0000 task.ti: ffff9c2c8bbc0000 [2642432.354173] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.354174] RSP: 0018:ffff9c2c8bbc3b68 EFLAGS: 00000246 [2642432.354175] RAX: 0000000000000000 RBX: ffff9c2c8bbc3b18 RCX: 0000000000990000 [2642432.354175] RDX: ffff9c2d9d75b880 RSI: 0000000000590101 RDI: ffff9c2a4a5112dc [2642432.354176] RBP: ffff9c2c8bbc3b68 R08: ffff9c2d9d85b880 R09: 0000000000000000 [2642432.354176] R10: ffff9c147036d800 R11: ffff9c2d9b324780 R12: ffff9c224efc5850 [2642432.354177] R13: ffff9c1fe1c69000 R14: 00000000cced4a80 R15: 0000000000000000 [2642432.354178] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2642432.354179] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.354179] CR2: 00000000006dee20 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.354180] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.354180] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.354181] PKRU: 00000000 [2642432.354181] Call Trace: [2642432.354184] [] queued_spin_lock_slowpath+0xb/0xf [2642432.354186] [] _raw_spin_lock+0x20/0x30 [2642432.354201] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.354216] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.354237] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.354254] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.354274] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.354299] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.354324] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.354346] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.354350] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.354370] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.354390] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.354392] [] ? __wake_up+0x13/0x20 [2642432.354412] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.354432] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.354433] [] kthread+0xd1/0xe0 [2642432.354435] [] ? insert_kthread_work+0x40/0x40 [2642432.354436] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.354438] [] ? insert_kthread_work+0x40/0x40 [2642432.354452] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.358117] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_ost00_114:26902] [2642432.358138] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.358158] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.358160] CPU: 20 PID: 26902 Comm: ll_ost00_114 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.358161] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.358161] task: ffff9c1444633150 ti: ffff9c145f230000 task.ti: ffff9c145f230000 [2642432.358164] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.358165] RSP: 0018:ffff9c145f233b68 EFLAGS: 00000246 [2642432.358166] RAX: 0000000000000000 RBX: ffff9c145f233b18 RCX: 0000000000a10000 [2642432.358166] RDX: ffff9c2d9d71b880 RSI: 0000000000490101 RDI: ffff9c2a4a5112dc [2642432.358167] RBP: ffff9c145f233b68 R08: ffff9c159f49b880 R09: 0000000000000000 [2642432.358168] R10: ffff9c147036d800 R11: ffff9c146eba4280 R12: ffff9c146b61c050 [2642432.358168] R13: ffff9c146a8fe400 R14: 00000000b3768c08 R15: 0000000000000000 [2642432.358169] FS: 0000000000000000(0000) GS:ffff9c159f480000(0000) knlGS:0000000000000000 [2642432.358170] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.358171] CR2: 00007f758dfdac3f CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.358171] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.358172] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.358172] PKRU: 00000000 [2642432.358172] Call Trace: [2642432.358175] [] queued_spin_lock_slowpath+0xb/0xf [2642432.358177] [] _raw_spin_lock+0x20/0x30 [2642432.358194] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.358209] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.358230] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.358248] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.358268] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.358293] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.358317] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.358340] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.358344] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.358364] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.358384] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.358386] [] ? __wake_up+0x13/0x20 [2642432.358406] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.358426] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.358427] [] kthread+0xd1/0xe0 [2642432.358429] [] ? insert_kthread_work+0x40/0x40 [2642432.358431] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.358432] [] ? insert_kthread_work+0x40/0x40 [2642432.358446] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.362107] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ll_ost01_096:24755] [2642432.362128] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.362148] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.362150] CPU: 21 PID: 24755 Comm: ll_ost01_096 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.362150] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.362151] task: ffff9c143921e2a0 ti: ffff9c1439b58000 task.ti: ffff9c1439b58000 [2642432.362154] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.362154] RSP: 0018:ffff9c1439b5bb68 EFLAGS: 00000246 [2642432.362155] RAX: 0000000000000000 RBX: ffff9c1439b5bb18 RCX: 0000000000a90000 [2642432.362156] RDX: ffff9c2d9d61b880 RSI: 0000000000090101 RDI: ffff9c2a4a5112dc [2642432.362156] RBP: ffff9c1439b5bb68 R08: ffff9c2d9d89b880 R09: 0000000000000000 [2642432.362157] R10: ffff9c147036d800 R11: ffff9c144c6a9c00 R12: ffff9c1f3ae40850 [2642432.362157] R13: ffff9c2ab62d9c00 R14: 000000007901e2bb R15: 0000000000000000 [2642432.362158] FS: 0000000000000000(0000) GS:ffff9c2d9d880000(0000) knlGS:0000000000000000 [2642432.362159] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.362159] CR2: 000000c001969010 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.362160] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.362161] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.362161] PKRU: 00000000 [2642432.362161] Call Trace: [2642432.362164] [] queued_spin_lock_slowpath+0xb/0xf [2642432.362166] [] _raw_spin_lock+0x20/0x30 [2642432.362181] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.362195] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.362216] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.362234] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.362254] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.362279] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.362303] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.362326] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.362329] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.362350] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.362370] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.362372] [] ? __wake_up+0x13/0x20 [2642432.362392] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.362411] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.362413] [] kthread+0xd1/0xe0 [2642432.362414] [] ? insert_kthread_work+0x40/0x40 [2642432.362416] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.362418] [] ? insert_kthread_work+0x40/0x40 [2642432.362432] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.366098] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ll_ost00_063:24727] [2642432.366119] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.366140] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.366142] CPU: 22 PID: 24727 Comm: ll_ost00_063 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.366142] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.366143] task: ffff9c148e8762a0 ti: ffff9c143159c000 task.ti: ffff9c143159c000 [2642432.366146] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.366147] RSP: 0018:ffff9c143159fb68 EFLAGS: 00000246 [2642432.366147] RAX: 0000000000000000 RBX: ffff9c143159fb18 RCX: 0000000000b10000 [2642432.366148] RDX: ffff9c159f41b880 RSI: 0000000000810101 RDI: ffff9c2a4a5112dc [2642432.366149] RBP: ffff9c143159fb68 R08: ffff9c159f4db880 R09: 0000000000000000 [2642432.366149] R10: ffff9c147036d800 R11: ffff9c14498b1a00 R12: ffff9c144ee48850 [2642432.366150] R13: ffff9c133e14cc00 R14: 00000000e13267e9 R15: 0000000000000000 [2642432.366151] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2642432.366151] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.366152] CR2: 00007f758c61ac80 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.366153] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.366153] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.366153] PKRU: 00000000 [2642432.366154] Call Trace: [2642432.366157] [] queued_spin_lock_slowpath+0xb/0xf [2642432.366159] [] _raw_spin_lock+0x20/0x30 [2642432.366176] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.366190] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.366211] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.366229] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.366249] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.366275] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.366300] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.366322] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.366326] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.366347] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.366367] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.366369] [] ? __wake_up+0x13/0x20 [2642432.366389] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.366409] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.366410] [] kthread+0xd1/0xe0 [2642432.366412] [] ? insert_kthread_work+0x40/0x40 [2642432.366414] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.366415] [] ? insert_kthread_work+0x40/0x40 [2642432.366429] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642432.370088] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_ost01_080:24715] [2642432.370109] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642432.370128] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642432.370130] CPU: 23 PID: 24715 Comm: ll_ost01_080 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642432.370130] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642432.370131] task: ffff9c152b23b150 ti: ffff9c142fef0000 task.ti: ffff9c142fef0000 [2642432.370134] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642432.370134] RSP: 0018:ffff9c142fef3b68 EFLAGS: 00000246 [2642432.370135] RAX: 0000000000000000 RBX: ffff9c142fef3b18 RCX: 0000000000b90000 [2642432.370136] RDX: ffff9c159f29b880 RSI: 0000000000210101 RDI: ffff9c2a4a5112dc [2642432.370136] RBP: ffff9c142fef3b68 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2642432.370137] R10: ffff9c147036d800 R11: ffff9c2c589bc900 R12: ffff9c1f3ae40050 [2642432.370137] R13: ffff9c2c1a2a2800 R14: 00000000af757e5f R15: 0000000000000000 [2642432.370138] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2642432.370139] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642432.370139] CR2: 000000c001941010 CR3: 000000172e636000 CR4: 00000000007607e0 [2642432.370140] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642432.370141] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642432.370141] PKRU: 00000000 [2642432.370141] Call Trace: [2642432.370144] [] queued_spin_lock_slowpath+0xb/0xf [2642432.370146] [] _raw_spin_lock+0x20/0x30 [2642432.370161] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642432.370176] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642432.370196] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642432.370214] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642432.370234] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642432.370259] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642432.370283] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642432.370305] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642432.370309] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642432.370329] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642432.370349] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642432.370351] [] ? __wake_up+0x13/0x20 [2642432.370370] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642432.370390] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642432.370392] [] kthread+0xd1/0xe0 [2642432.370393] [] ? insert_kthread_work+0x40/0x40 [2642432.370395] [] ret_from_fork_nospec_begin+0x7/0x21 [2642432.370396] [] ? insert_kthread_work+0x40/0x40 [2642432.370410] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642442.817790] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2642442.895090] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2642442.968035] CPU: 1 PID: 24810 Comm: ll_ost01_128 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2642442.981435] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2642442.989849] task: ffff9c144d355230 ti: ffff9c1454970000 task.ti: ffff9c1454970000 [2642442.998173] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2642443.008974] RSP: 0018:ffff9c1454973b68 EFLAGS: 00000246 [2642443.015154] RAX: 0000000000000000 RBX: ffff9c1454973b18 RCX: 0000000000090000 [2642443.023151] RDX: ffff9c2d9d81b880 RSI: 0000000000890101 RDI: ffff9c2a4a5112dc [2642443.031148] RBP: ffff9c1454973b68 R08: ffff9c2d9d61b880 R09: 0000000000000000 [2642443.039154] R10: ffff9c147036d800 R11: ffff9c144a5176c0 R12: ffff9c1f3ae45850 [2642443.047161] R13: ffff9c21e6b33000 R14: 00000000752d18fc R15: 0000000000000000 [2642443.055167] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2642443.064140] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2642443.070781] CR2: 00007f77d9493000 CR3: 000000172e636000 CR4: 00000000007607e0 [2642443.078803] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2642443.086826] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2642443.094849] PKRU: 00000000 [2642443.098456] Call Trace: [2642443.101804] [] queued_spin_lock_slowpath+0xb/0xf [2642443.108961] [] _raw_spin_lock+0x20/0x30 [2642443.115347] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2642443.122775] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2642443.130381] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2642443.137987] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2642443.145942] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2642443.154333] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2642443.161334] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2642443.169116] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2642443.177566] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2642443.185523] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2642443.194089] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2642443.201771] [] ? __wake_up+0x13/0x20 [2642443.207914] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2642443.215081] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2642443.223344] [] kthread+0xd1/0xe0 [2642443.229085] [] ? insert_kthread_work+0x40/0x40 [2642443.236027] [] ret_from_fork_nospec_begin+0x7/0x21 [2642443.243296] [] ? insert_kthread_work+0x40/0x40 [2642443.250213] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2642443.273481] Lustre: f2-OST0014: Client f2-MDT0003-mdtlov_UUID (at 10.10.33.9@o2ib2) reconnecting [2642443.278786] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.33.8@o2ib2: 5570 seconds [2642443.278802] LNetError: 18738:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.10.33.33@o2ib2 added to recovery queue. Health = 900 [2642443.292747] LNetError: 18744:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.33.53@o2ib2 added to recovery queue. Health = 900 [2642443.292750] LNetError: 18744:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 13 previous similar messages [2642443.304399] LustreError: 18739:0:(events.c:450:server_bulk_callback()) event type 3, status -5, desc ffff9c146addfc00 [2642443.304603] LustreError: 18739:0:(events.c:450:server_bulk_callback()) event type 3, status -5, desc ffff9c133e14e400 [2642444.270205] LustreError: 25902:0:(ldlm_lib.c:3277:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff9c146bd28050 x1655472326360656/t0(0) o4->4f799828-a47d-efca-07d0-28cee6fe1648@10.10.32.135@o2ib2:305/0 lens 608/448 e 0 to 0 dl 1585759270 ref 1 fl Interpret:/0/0 rc 0/0 [2642444.296545] LustreError: 25902:0:(ldlm_lib.c:3277:target_bulk_io()) Skipped 1 previous similar message [2642444.296718] LNetError: 17456:0:(lib-move.c:4761:lnet_create_reply_msg()) 10.10.33.33@o2ib2: Dropping REPLY from 12345-10.10.32.135@o2ib2 for inactive MD ffff9c146addd000 [2642444.296720] LNetError: 17456:0:(lib-move.c:4761:lnet_create_reply_msg()) Skipped 1 previous similar message [2642444.296725] LNetError: 17456:0:(o2iblnd_cb.c:1727:kiblnd_send()) Can't create reply for GET -> 10.10.32.135@o2ib2 [2642444.296727] LNetError: 17456:0:(o2iblnd_cb.c:1727:kiblnd_send()) Skipped 1 previous similar message [2642444.296733] LustreError: 17456:0:(events.c:450:server_bulk_callback()) event type 5, status -5, desc ffff9c146addfc00 [2642444.296766] LNetError: 17456:0:(lib-move.c:2955:lnet_resend_pending_msgs_locked()) Error sending GET to 12345-10.10.32.135@o2ib2: -125 [2642444.296775] LustreError: 17456:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9c133e14e400 [2642444.296829] Lustre: f2-OST0014: Bulk IO write error with 4f799828-a47d-efca-07d0-28cee6fe1648 (at 10.10.32.135@o2ib2), client will retry: rc = -110 [2642444.296830] Lustre: Skipped 21 previous similar messages [2642465.848542] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000401:4561907 to 0x7c0000401:4561985 [2642465.865848] Lustre: f2-OST0038: deleting orphan objects from 0x940000401:4563731 to 0x940000401:4563809 [2642467.557407] Lustre: f2-OST0014: deleting orphan objects from 0x0:6899534 to 0x0:6899649 [2642467.574583] Lustre: f2-OST0038: deleting orphan objects from 0x0:6898230 to 0x0:6898346 [2642467.923322] Lustre: f2-OST0038: deleting orphan objects from 0x940000400:5470450 to 0x940000400:5470481 [2642467.938755] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000400:5468949 to 0x7c0000400:5468980 [2642473.514258] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000402:3497381 to 0x7c0000402:3497441 [2642473.533320] Lustre: f2-OST0038: deleting orphan objects from 0x940000402:3495591 to 0x940000402:3495745 [2642491.931357] Lustre: f2-OST0014: Client e4d445a6-0133-48b1-452f-c41e433a63a7 (at 10.10.33.50@o2ib2) reconnecting [2642491.942377] Lustre: Skipped 23 previous similar messages [2642544.503989] Lustre: Unmounted f2-client [2642614.557865] LustreError: 22703:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 149s: evicting client at 10.10.33.4@o2ib2 ns: filter-f2-OST0014_UUID lock: ffff9c25579c5100/0x760b200c5ba8a0a lrc: 3/0,0 mode: PR/PR res: [0x7c0000401:0x4598b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x60000400010020 nid: 10.10.33.4@o2ib2 remote: 0x7d64ba87cbabaf49 expref: 2006 pid: 22881 timeout: 2649043 lvb_type: 1 [2642614.603043] LustreError: 22703:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 13 previous similar messages [2642716.656360] Lustre: f2-OST0014: Connection restored to d28ef103-ecee-6bbc-6acb-e927a712e686 (at 10.10.33.4@o2ib2) [2642716.667653] Lustre: Skipped 30 previous similar messages [2642972.235074] Lustre: f2-OST0038: Client 3964c497-b39d-8564-eb85-54a400de46f9 (at 10.10.33.4@o2ib2) reconnecting [2644698.102699] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2644698.113902] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 8 previous similar messages [2644698.125064] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.140@o2ib2 (6): c: 5, oc: 0, rc: 8 [2644698.138919] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 8 previous similar messages [2644703.138451] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.144@o2ib2 added to recovery queue. Health = 900 [2644703.153426] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 6 previous similar messages [2644703.165200] LustreError: 26127:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(1015512) req@ffff9c146731f050 x1655452889573264/t0(0) o4->a1725fdf-452e-7b6d-04bc-4e663aa1a39c@352@gni5:269/0 lens 608/448 e 0 to 0 dl 1585761499 ref 1 fl Interpret:/0/0 rc 0/0 [2644703.165224] Lustre: f2-OST0014: Bulk IO write error with 06f00903-69de-ae9a-7a26-ab4a5726eff9 (at 1528@gni5), client will retry: rc = -110 [2644703.165225] Lustre: Skipped 1 previous similar message [2644703.212402] LustreError: 26127:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 22 previous similar messages [2644708.152254] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.167@o2ib2 added to recovery queue. Health = 900 [2644708.152261] LustreError: 26134:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(976520) req@ffff9c146aa45050 x1661790504789440/t0(0) o4->38792623-3a1e-84c6-ec13-1a52e12de9d8@317@gni4:269/0 lens 608/448 e 0 to 0 dl 1585761499 ref 1 fl Interpret:/0/0 rc 0/0 [2644708.152344] Lustre: f2-OST0038: Bulk IO write error with 38792623-3a1e-84c6-ec13-1a52e12de9d8 (at 317@gni4), client will retry: rc = -110 [2644708.152345] Lustre: Skipped 2 previous similar messages [2644708.214397] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 3 previous similar messages [2644708.226314] LustreError: 23731:0:(ldlm_lib.c:3292:target_bulk_io()) @@@ truncated bulk READ 0(1048576) req@ffff9c12807e8050 x1661790546385056/t0(0) o3->9f4adbc6-10e8-b78a-317e-1f207f842b9a@1536@gni4:270/0 lens 608/440 e 0 to 0 dl 1585761500 ref 1 fl Interpret:/0/0 rc 0/0 [2644708.226355] Lustre: f2-OST0038: Bulk IO read error with 7f428279-6981-23f8-5b91-a0c032495d49 (at 1537@gni4), client will retry: rc -110 [2644708.226356] Lustre: f2-OST0038: Bulk IO read error with 7f428279-6981-23f8-5b91-a0c032495d49 (at 1537@gni4), client will retry: rc -110 [2644708.226357] Lustre: Skipped 3 previous similar messages [2644708.226358] Lustre: Skipped 3 previous similar messages [2644708.293784] LustreError: 23731:0:(ldlm_lib.c:3292:target_bulk_io()) Skipped 5 previous similar messages [2644713.213954] LustreError: 25981:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(131072) req@ffff9c145b1d8050 x1661790580339152/t0(0) o4->949f9444-0f28-0543-1812-0720f4a9101e@2378@gni4:277/0 lens 608/448 e 0 to 0 dl 1585761507 ref 1 fl Interpret:/0/0 rc 0/0 [2644713.213968] Lustre: f2-OST0014: Bulk IO write error with b1fd66b5-1bef-b7bd-f539-ef0c914ea11c (at 1979@gni4), client will retry: rc = -110 [2644713.213969] Lustre: Skipped 11 previous similar messages [2644713.262087] LustreError: 25981:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 16 previous similar messages [2644714.063873] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2644714.075155] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 10 previous similar messages [2644714.086507] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.152@o2ib2 (22): c: 4, oc: 0, rc: 8 [2644714.100675] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 10 previous similar messages [2644718.201805] LustreError: 38466:0:(ldlm_lib.c:3292:target_bulk_io()) @@@ truncated bulk READ 0(240588) req@ffff9c1467319050 x1661790403845328/t0(0) o3->1b3f5326-7b7a-e140-1b4e-cc53742961f3@114@gni4:280/0 lens 608/440 e 0 to 0 dl 1585761510 ref 1 fl Interpret:/0/0 rc 0/0 [2644718.201818] LustreError: 25945:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(1048576) req@ffff9c146b7dc850 x1661790342296032/t0(0) o4->bb73a82d-2e24-355b-e3df-7245b14878ca@1716@gni4:281/0 lens 608/448 e 0 to 0 dl 1585761511 ref 1 fl Interpret:/0/0 rc 0/0 [2644718.201819] Lustre: f2-OST0038: Bulk IO read error with a58260d2-886a-fccb-0c9b-b6821d0978f9 (at 108@gni4), client will retry: rc -110 [2644718.201820] Lustre: Skipped 4 previous similar messages [2644718.201836] Lustre: f2-OST0038: Bulk IO write error with bb73a82d-2e24-355b-e3df-7245b14878ca (at 1716@gni4), client will retry: rc = -110 [2644718.201837] Lustre: Skipped 5 previous similar messages [2644718.297319] LustreError: 38466:0:(ldlm_lib.c:3292:target_bulk_io()) Skipped 1 previous similar message [2644718.717530] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ll_ost01_089:24734] [2644718.726507] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [ll_ost01_065:24691] [2644718.726541] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.726574] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.726577] CPU: 3 PID: 24691 Comm: ll_ost01_065 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.726577] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.726578] task: ffff9c2c83900000 ti: ffff9c1445480000 task.ti: ffff9c1445480000 [2644718.726583] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.726585] RSP: 0018:ffff9c1445483bc8 EFLAGS: 00000246 [2644718.726586] RAX: 0000000000000000 RBX: ffff9c1445483b68 RCX: 0000000000190000 [2644718.726587] RDX: ffff9c2d9d71b880 RSI: 0000000000490000 RDI: ffff9c2a4a5112dc [2644718.726587] RBP: ffff9c1445483bc8 R08: ffff9c2d9d65b880 R09: 0000000000000000 [2644718.726588] R10: ffff9c23ed3a69c0 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.726588] R13: ffff9c23ed3a69c0 R14: ffff9c23ed3a69c0 R15: 000000003999acfb [2644718.726589] FS: 0000000000000000(0000) GS:ffff9c2d9d640000(0000) knlGS:0000000000000000 [2644718.726590] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.726590] CR2: 000000c0022a3000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644718.726591] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.726592] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.726592] PKRU: 00000000 [2644718.726592] Call Trace: [2644718.726598] [] queued_spin_lock_slowpath+0xb/0xf [2644718.726602] [] _raw_spin_lock+0x20/0x30 [2644718.726613] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.726646] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.726668] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.726697] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.726722] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.726744] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.726753] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.726774] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.726794] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.726814] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.726833] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.726836] [] kthread+0xd1/0xe0 [2644718.726837] [] ? insert_kthread_work+0x40/0x40 [2644718.726840] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.726841] [] ? insert_kthread_work+0x40/0x40 [2644718.726856] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.733490] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [ll_ost01_122:24802] [2644718.733515] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.733538] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.733540] CPU: 5 PID: 24802 Comm: ll_ost01_122 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.733541] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.733542] task: ffff9c14540dc1c0 ti: ffff9c1454908000 task.ti: ffff9c1454908000 [2644718.733546] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.733546] RSP: 0018:ffff9c145490bbc8 EFLAGS: 00000246 [2644718.733547] RAX: 0000000000000000 RBX: ffff9c145490bb68 RCX: 0000000000290000 [2644718.733548] RDX: ffff9c2d9d89b880 RSI: 0000000000a90000 RDI: ffff9c2a4a5112dc [2644718.733548] RBP: ffff9c145490bbc8 R08: ffff9c2d9d69b880 R09: 0000000000000000 [2644718.733549] R10: ffff9c26ba4d6c00 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.733549] R13: ffff9c26ba4d6c00 R14: ffff9c26ba4d6c00 R15: 000000002a770847 [2644718.733550] FS: 0000000000000000(0000) GS:ffff9c2d9d680000(0000) knlGS:0000000000000000 [2644718.733551] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.733551] CR2: 0000000001cc071c CR3: 000000190da10000 CR4: 00000000007607e0 [2644718.733552] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.733553] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.733553] PKRU: 00000000 [2644718.733553] Call Trace: [2644718.733557] [] queued_spin_lock_slowpath+0xb/0xf [2644718.733560] [] _raw_spin_lock+0x20/0x30 [2644718.733566] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.733591] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.733611] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.733638] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.733662] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.733685] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.733690] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.733710] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.733729] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.733748] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2644718.733767] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2644718.733786] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.733806] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.733808] [] kthread+0xd1/0xe0 [2644718.733810] [] ? insert_kthread_work+0x40/0x40 [2644718.733811] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.733813] [] ? insert_kthread_work+0x40/0x40 [2644718.733827] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.742468] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ll_ost01_147:25275] [2644718.742491] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.742512] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.742514] CPU: 7 PID: 25275 Comm: ll_ost01_147 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.742514] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.742515] task: ffff9c2c8b530000 ti: ffff9c2c642fc000 task.ti: ffff9c2c642fc000 [2644718.742518] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.742519] RSP: 0018:ffff9c2c642ffbc8 EFLAGS: 00000246 [2644718.742519] RAX: 0000000000000000 RBX: ffff9c2c642ffb68 RCX: 0000000000390000 [2644718.742520] RDX: ffff9c2d9d75b880 RSI: 0000000000590000 RDI: ffff9c2a4a5112dc [2644718.742520] RBP: ffff9c2c642ffbc8 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2644718.742521] R10: ffff9c2798485e80 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.742521] R13: ffff9c2798485e80 R14: ffff9c2798485e80 R15: 0000000004f9a6be [2644718.742522] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2644718.742523] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.742524] CR2: 000000c00135d000 CR3: 0000002f0b7ba000 CR4: 00000000007607e0 [2644718.742524] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.742525] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.742525] PKRU: 00000000 [2644718.742526] Call Trace: [2644718.742529] [] queued_spin_lock_slowpath+0xb/0xf [2644718.742531] [] _raw_spin_lock+0x20/0x30 [2644718.742537] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.742561] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.742581] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.742607] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.742632] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.742654] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.742658] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.742679] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.742699] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.742718] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.742738] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.742739] [] kthread+0xd1/0xe0 [2644718.742741] [] ? insert_kthread_work+0x40/0x40 [2644718.742742] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.742744] [] ? insert_kthread_work+0x40/0x40 [2644718.742758] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.750450] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ll_ost01_030:23816] [2644718.750471] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.750491] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.750493] CPU: 9 PID: 23816 Comm: ll_ost01_030 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.750493] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.750494] task: ffff9c2c81e162a0 ti: ffff9c2ce5be0000 task.ti: ffff9c2ce5be0000 [2644718.750497] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.750498] RSP: 0018:ffff9c2ce5be3bc8 EFLAGS: 00000246 [2644718.750498] RAX: 0000000000000000 RBX: ffff9c2ce5be3b68 RCX: 0000000000490000 [2644718.750499] RDX: ffff9c2d9d69b880 RSI: 0000000000290000 RDI: ffff9c2a4a5112dc [2644718.750499] RBP: ffff9c2ce5be3bc8 R08: ffff9c2d9d71b880 R09: 0000000000000000 [2644718.750500] R10: ffff9c180fb33a80 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.750500] R13: ffff9c180fb33a80 R14: ffff9c180fb33a80 R15: 000000006642cb2a [2644718.750501] FS: 0000000000000000(0000) GS:ffff9c2d9d700000(0000) knlGS:0000000000000000 [2644718.750502] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.750503] CR2: 000000c0022a7000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644718.750503] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.750504] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.750504] PKRU: 00000000 [2644718.750505] Call Trace: [2644718.750507] [] queued_spin_lock_slowpath+0xb/0xf [2644718.750509] [] _raw_spin_lock+0x20/0x30 [2644718.750514] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.750534] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.750554] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.750580] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.750604] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.750626] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.750630] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.750651] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.750670] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.750689] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2644718.750708] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2644718.750727] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.750746] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.750748] [] kthread+0xd1/0xe0 [2644718.750749] [] ? insert_kthread_work+0x40/0x40 [2644718.750751] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.750752] [] ? insert_kthread_work+0x40/0x40 [2644718.750767] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.758432] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ll_ost01_058:24679] [2644718.758453] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.758473] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.758474] CPU: 11 PID: 24679 Comm: ll_ost01_058 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.758475] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.758476] task: ffff9c152eeb9070 ti: ffff9c1424a00000 task.ti: ffff9c1424a00000 [2644718.758478] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.758479] RSP: 0018:ffff9c1424a03bc8 EFLAGS: 00000246 [2644718.758480] RAX: 0000000000000000 RBX: ffff9c1424a03b68 RCX: 0000000000590000 [2644718.758480] RDX: ffff9c2d9d65b880 RSI: 0000000000190000 RDI: ffff9c2a4a5112dc [2644718.758481] RBP: ffff9c1424a03bc8 R08: ffff9c2d9d75b880 R09: 0000000000000000 [2644718.758481] R10: ffff9c1db7c7b600 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.758482] R13: ffff9c1db7c7b600 R14: ffff9c1db7c7b600 R15: 000000002c58153d [2644718.758483] FS: 0000000000000000(0000) GS:ffff9c2d9d740000(0000) knlGS:0000000000000000 [2644718.758484] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.758484] CR2: 000000c00215b000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644718.758485] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.758485] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.758486] PKRU: 00000000 [2644718.758486] Call Trace: [2644718.758489] [] queued_spin_lock_slowpath+0xb/0xf [2644718.758491] [] _raw_spin_lock+0x20/0x30 [2644718.758496] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.758516] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.758537] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.758563] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.758587] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.758610] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.758614] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.758634] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.758654] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.758656] [] ? queued_spin_lock_slowpath+0xb/0xf [2644718.758657] [] ? _raw_spin_lock+0x20/0x30 [2644718.758677] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.758697] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.758698] [] kthread+0xd1/0xe0 [2644718.758700] [] ? insert_kthread_work+0x40/0x40 [2644718.758702] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.758703] [] ? insert_kthread_work+0x40/0x40 [2644718.758717] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.766410] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ll_ost01_109:24775] [2644718.766431] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.766451] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.766453] CPU: 13 PID: 24775 Comm: ll_ost01_109 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.766453] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.766454] task: ffff9c143ae05230 ti: ffff9c143be18000 task.ti: ffff9c143be18000 [2644718.766457] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.766457] RSP: 0018:ffff9c143be1bb90 EFLAGS: 00000246 [2644718.766458] RAX: 0000000000000000 RBX: ffff9c143be1badc RCX: 0000000000690000 [2644718.766459] RDX: ffff9c2d9d61b880 RSI: 0000000000090000 RDI: ffff9c2a4a5112dc [2644718.766459] RBP: ffff9c143be1bb90 R08: ffff9c2d9d79b880 R09: 0000000000000000 [2644718.766460] R10: ffff9c27acb07100 R11: ffff9c2ad7be0000 R12: 0000000000000002 [2644718.766460] R13: ffff9c1f3c4a1680 R14: ffff9c143be1bbd8 R15: 00000000047e5fff [2644718.766461] FS: 0000000000000000(0000) GS:ffff9c2d9d780000(0000) knlGS:0000000000000000 [2644718.766462] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.766463] CR2: 000000c0013ff010 CR3: 000000190da10000 CR4: 00000000007607e0 [2644718.766463] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.766464] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.766464] PKRU: 00000000 [2644718.766465] Call Trace: [2644718.766467] [] queued_spin_lock_slowpath+0xb/0xf [2644718.766469] [] _raw_spin_lock+0x20/0x30 [2644718.766486] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644718.766501] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644718.766518] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644718.766539] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.766564] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.766589] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.766611] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.766615] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.766635] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.766655] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.766674] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2644718.766692] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2644718.766711] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.766730] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.766732] [] kthread+0xd1/0xe0 [2644718.766734] [] ? insert_kthread_work+0x40/0x40 [2644718.766735] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.766737] [] ? insert_kthread_work+0x40/0x40 [2644718.766751] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.775388] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ldlm_cn01_021:52157] [2644718.775410] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.775430] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.775432] CPU: 15 PID: 52157 Comm: ldlm_cn01_021 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.775432] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.775433] task: ffff9c2a64c720e0 ti: ffff9c284078c000 task.ti: ffff9c284078c000 [2644718.775452] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2644718.775453] RSP: 0018:ffff9c284078fac0 EFLAGS: 00000202 [2644718.775453] RAX: ffff9c11880e0240 RBX: ffff9c284078fa70 RCX: 000000000000000e [2644718.775454] RDX: 0000000000000004 RSI: 000000000480e000 RDI: ffff9c146e162000 [2644718.775454] RBP: ffff9c284078fb48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2644718.775455] R10: 00000000047e5000 R11: 000000000480e000 R12: ffff9c0f107f1480 [2644718.775455] R13: 0000000000000000 R14: ffff9c0c4c108720 R15: ffff9c0f107f1480 [2644718.775456] FS: 0000000000000000(0000) GS:ffff9c2d9d7c0000(0000) knlGS:0000000000000000 [2644718.775457] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.775458] CR2: 00007f9d0f438000 CR3: 0000002fc04f6000 CR4: 00000000007607e0 [2644718.775458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.775459] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.775459] PKRU: 00000000 [2644718.775459] Call Trace: [2644718.775477] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2644718.775493] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2644718.775508] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2644718.775514] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2644718.775528] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2644718.775543] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644718.775559] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2644718.775576] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2644718.775593] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2644718.775613] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.775633] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.775636] [] ? __wake_up+0x13/0x20 [2644718.775656] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.775676] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.775677] [] kthread+0xd1/0xe0 [2644718.775679] [] ? insert_kthread_work+0x40/0x40 [2644718.775681] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.775682] [] ? insert_kthread_work+0x40/0x40 [2644718.775696] Code: b8 00 00 00 74 2f 48 8b 50 60 49 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 00 49 39 c7 74 e1 8b 90 98 00 00 00 <44> 85 24 95 00 01 5a c1 75 c1 83 c1 01 48 89 f7 48 8b 90 d0 00 [2644718.783369] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ll_ost01_016:23724] [2644718.783389] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.783409] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.783410] CPU: 17 PID: 23724 Comm: ll_ost01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.783411] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.783412] task: ffff9c148e91d230 ti: ffff9c1577718000 task.ti: ffff9c1577718000 [2644718.783415] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.783415] RSP: 0018:ffff9c157771bbc8 EFLAGS: 00000246 [2644718.783416] RAX: 0000000000000000 RBX: ffff9c157771bb68 RCX: 0000000000890000 [2644718.783417] RDX: ffff9c2d9d85b880 RSI: 0000000000790001 RDI: ffff9c2a4a5112dc [2644718.783417] RBP: ffff9c157771bbc8 R08: ffff9c2d9d81b880 R09: 0000000000000000 [2644718.783418] R10: ffff9c2d0967e540 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.783418] R13: ffff9c2d0967e540 R14: ffff9c2d0967e540 R15: 00000000f6a80802 [2644718.783419] FS: 0000000000000000(0000) GS:ffff9c2d9d800000(0000) knlGS:0000000000000000 [2644718.783420] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.783420] CR2: 000000c001353010 CR3: 000000172e636000 CR4: 00000000007607e0 [2644718.783421] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.783422] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.783422] PKRU: 00000000 [2644718.783422] Call Trace: [2644718.783425] [] queued_spin_lock_slowpath+0xb/0xf [2644718.783427] [] _raw_spin_lock+0x20/0x30 [2644718.783432] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.783451] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.783471] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.783497] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.783521] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.783543] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.783547] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.783567] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.783587] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.783589] [] ? __wake_up+0x13/0x20 [2644718.783608] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.783628] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.783630] [] kthread+0xd1/0xe0 [2644718.783631] [] ? insert_kthread_work+0x40/0x40 [2644718.783633] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.783634] [] ? insert_kthread_work+0x40/0x40 [2644718.783648] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.791351] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ll_ost01_024:23782] [2644718.791372] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.791391] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.791393] CPU: 19 PID: 23782 Comm: ll_ost01_024 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.791393] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.791394] task: ffff9c2c946d8000 ti: ffff9c2d1a74c000 task.ti: ffff9c2d1a74c000 [2644718.791396] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2644718.791397] RSP: 0018:ffff9c2d1a74fbc8 EFLAGS: 00000202 [2644718.791398] RAX: 0000000000000001 RBX: ffff9c2d1a74fb68 RCX: 0000000000990000 [2644718.791398] RDX: 0000000000690001 RSI: 0000000000790000 RDI: ffff9c2a4a5112dc [2644718.791399] RBP: ffff9c2d1a74fbc8 R08: ffff9c2d9d85b880 R09: ffff9c2d9d81b880 [2644718.791399] R10: ffff9c2bfbd0bf00 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.791400] R13: ffff9c2bfbd0bf00 R14: ffff9c2bfbd0bf00 R15: 0000000050799a69 [2644718.791401] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2644718.791402] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.791402] CR2: 00007f76f4e82000 CR3: 0000001722746000 CR4: 00000000007607e0 [2644718.791403] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.791403] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.791404] PKRU: 00000000 [2644718.791404] Call Trace: [2644718.791407] [] queued_spin_lock_slowpath+0xb/0xf [2644718.791409] [] _raw_spin_lock+0x20/0x30 [2644718.791414] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.791433] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.791453] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.791479] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.791503] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.791525] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.791529] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.791549] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.791569] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.791588] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.791608] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.791609] [] kthread+0xd1/0xe0 [2644718.791611] [] ? insert_kthread_work+0x40/0x40 [2644718.791613] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.791614] [] ? insert_kthread_work+0x40/0x40 [2644718.791628] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2644718.799330] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ll_ost01_141:25269] [2644718.799351] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.799370] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.799372] CPU: 21 PID: 25269 Comm: ll_ost01_141 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.799372] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.799373] task: ffff9c2d9bb741c0 ti: ffff9c2c642b8000 task.ti: ffff9c2c642b8000 [2644718.799376] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.799377] RSP: 0018:ffff9c2c642bbbc8 EFLAGS: 00000246 [2644718.799377] RAX: 0000000000000000 RBX: ffff9c2c642bbb68 RCX: 0000000000a90000 [2644718.799378] RDX: ffff9c2d9d81b880 RSI: 0000000000890001 RDI: ffff9c2a4a5112dc [2644718.799378] RBP: ffff9c2c642bbbc8 R08: ffff9c2d9d89b880 R09: 0000000000000000 [2644718.799379] R10: ffff9c1fb5156540 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644718.799380] R13: ffff9c1fb5156540 R14: ffff9c1fb5156540 R15: 00000000b80a40b8 [2644718.799381] FS: 0000000000000000(0000) GS:ffff9c2d9d880000(0000) knlGS:0000000000000000 [2644718.799381] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.799382] CR2: 0000000001b700a8 CR3: 000000172e636000 CR4: 00000000007607e0 [2644718.799383] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.799383] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.799383] PKRU: 00000000 [2644718.799384] Call Trace: [2644718.799387] [] queued_spin_lock_slowpath+0xb/0xf [2644718.799388] [] _raw_spin_lock+0x20/0x30 [2644718.799394] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644718.799412] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644718.799433] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.799458] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.799483] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.799505] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.799509] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.799529] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.799548] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.799550] [] ? __wake_up+0x13/0x20 [2644718.799570] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.799589] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.799591] [] kthread+0xd1/0xe0 [2644718.799593] [] ? insert_kthread_work+0x40/0x40 [2644718.799594] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.799596] [] ? insert_kthread_work+0x40/0x40 [2644718.799610] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644718.807311] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_ost01_128:24810] [2644718.807332] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644718.807351] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644718.807353] CPU: 23 PID: 24810 Comm: ll_ost01_128 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644718.807353] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644718.807354] task: ffff9c144d355230 ti: ffff9c1454970000 task.ti: ffff9c1454970000 [2644718.807357] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644718.807358] RSP: 0018:ffff9c1454973b90 EFLAGS: 00000246 [2644718.807358] RAX: 0000000000000000 RBX: ffff9c1454973adc RCX: 0000000000b90000 [2644718.807359] RDX: ffff9c2d9d6db880 RSI: 0000000000390000 RDI: ffff9c2a4a5112dc [2644718.807359] RBP: ffff9c1454973b90 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2644718.807360] R10: ffff9c2a52155500 R11: ffff9c207c150000 R12: 0000000000000002 [2644718.807361] R13: ffff9c1ee2fc2ac0 R14: ffff9c1454973bd8 R15: 000000000480efff [2644718.807362] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2644718.807362] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644718.807363] CR2: 000000c0013f7010 CR3: 0000001724304000 CR4: 00000000007607e0 [2644718.807363] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644718.807364] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644718.807364] PKRU: 00000000 [2644718.807365] Call Trace: [2644718.807368] [] queued_spin_lock_slowpath+0xb/0xf [2644718.807369] [] _raw_spin_lock+0x20/0x30 [2644718.807385] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644718.807400] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644718.807417] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644718.807438] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644718.807463] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644718.807488] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644718.807510] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644718.807514] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644718.807534] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644718.807554] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644718.807573] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2644718.807592] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2644718.807611] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644718.807630] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644718.807632] [] kthread+0xd1/0xe0 [2644718.807634] [] ? insert_kthread_work+0x40/0x40 [2644718.807635] [] ret_from_fork_nospec_begin+0x7/0x21 [2644718.807637] [] ? insert_kthread_work+0x40/0x40 [2644718.807651] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644723.189664] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.152@o2ib2 added to recovery queue. Health = 900 [2644723.189667] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 5 previous similar messages [2644723.189679] LustreError: 26126:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(1048576) req@ffff9c14f0207050 x1655452191535328/t0(0) o4->f9b8a6a0-23c1-31b9-8d5d-7fe092ef9759@216@gni5:284/0 lens 608/448 e 0 to 0 dl 1585761514 ref 1 fl Interpret:/0/0 rc 0/0 [2644723.189681] LustreError: 26126:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 4 previous similar messages [2644723.189697] Lustre: f2-OST0038: Bulk IO write error with f9b8a6a0-23c1-31b9-8d5d-7fe092ef9759 (at 216@gni5), client will retry: rc = -110 [2644723.189697] Lustre: Skipped 4 previous similar messages [2644723.731507] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644723.808913] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644723.882463] CPU: 1 PID: 24734 Comm: ll_ost01_089 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644723.896103] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644723.904636] task: ffff9c14e470b150 ti: ffff9c14322f8000 task.ti: ffff9c14322f8000 [2644723.913092] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644723.924004] RSP: 0018:ffff9c14322fbb90 EFLAGS: 00000246 [2644723.930306] RAX: 0000000000000000 RBX: ffff9c14322fbadc RCX: 0000000000090000 [2644723.938434] RDX: ffff9c2d9d8db880 RSI: 0000000000b90000 RDI: ffff9c2a4a5112dc [2644723.946560] RBP: ffff9c14322fbb90 R08: ffff9c2d9d61b880 R09: 0000000000000000 [2644723.954686] R10: ffff9c25f6829500 R11: ffff9c1f8cf40000 R12: 0000000000000002 [2644723.962813] R13: ffff9c2be4f9da00 R14: ffff9c14322fbbd8 R15: 000000000480efff [2644723.970941] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2644723.980019] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644723.986764] CR2: 00007f77d9493000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644723.994897] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644724.003025] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644724.011151] PKRU: 00000000 [2644724.014860] Call Trace: [2644724.018303] [] queued_spin_lock_slowpath+0xb/0xf [2644724.025558] [] _raw_spin_lock+0x20/0x30 [2644724.032058] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644724.039940] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644724.047560] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644724.055613] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644724.064089] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644724.071170] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644724.079011] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644724.087514] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644724.095514] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644724.104106] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644724.111818] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2644724.120472] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2644724.128426] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644724.135586] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644724.143847] [] kthread+0xd1/0xe0 [2644724.149589] [] ? insert_kthread_work+0x40/0x40 [2644724.156530] [] ret_from_fork_nospec_begin+0x7/0x21 [2644724.163800] [] ? insert_kthread_work+0x40/0x40 [2644724.170716] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644724.192741] LustreError: 18745:0:(events.c:450:server_bulk_callback()) event type 5, status -103, desc ffff9c28a23de400 [2644724.204363] LustreError: 26122:0:(ldlm_lib.c:3283:target_bulk_io()) @@@ network error on bulk WRITE req@ffff9c145b05d050 x1655452378266224/t0(0) o4->3609d1aa-6a24-8203-a913-2e9057577c3f@1112@gni5:268/0 lens 608/448 e 0 to 0 dl 1585761498 ref 1 fl Interpret:/0/0 rc 0/0 [2644724.204844] Lustre: f2-OST0014: Client f2-MDT0003-mdtlov_UUID (at 10.10.33.9@o2ib2) reconnecting [2644724.204861] Lustre: f2-OST0014: Connection restored to f2-MDT0003-mdtlov_UUID (at 10.10.33.9@o2ib2) [2644724.204863] Lustre: Skipped 1 previous similar message [2644724.205548] LustreError: 18743:0:(events.c:450:server_bulk_callback()) event type 5, status -103, desc ffff9c1fe1c6ac00 [2644724.267403] LustreError: 26122:0:(ldlm_lib.c:3283:target_bulk_io()) Skipped 1 previous similar message [2644751.487712] Lustre: f2-OST0014: Client f2-MDT0001-mdtlov_UUID (at 10.10.33.7@o2ib2) reconnecting [2644751.487809] Lustre: f2-OST0038: deleting orphan objects from 0x940000401:4564541 to 0x940000401:4564577 [2644751.507582] Lustre: Skipped 6 previous similar messages [2644751.513716] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000401:4562717 to 0x7c0000401:4562753 [2644754.217656] Lustre: f2-OST0038: deleting orphan objects from 0x0:6898763 to 0x0:6898794 [2644754.228425] Lustre: f2-OST0014: deleting orphan objects from 0x0:6900065 to 0x0:6900097 [2644756.855000] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000402:3497652 to 0x7c0000402:3497825 [2644756.874275] Lustre: f2-OST0038: deleting orphan objects from 0x940000402:3495961 to 0x940000402:3496065 [2644788.894733] Lustre: f2-OST0038: Client 88d1ee8e-8f41-fb32-9de1-07175f6391f8 (at 1213@gni5) reconnecting [2644788.904993] Lustre: Skipped 169 previous similar messages [2644800.897925] Lustre: f2-OST0014: Connection restored to be3261cd-2144-625f-6885-6b9352d61b43 (at 1711@gni4) [2644800.908398] Lustre: Skipped 213 previous similar messages [2644802.847843] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2644802.858840] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.150@o2ib2 (6): c: 6, oc: 0, rc: 8 [2644808.301583] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.154@o2ib2 added to recovery queue. Health = 900 [2644808.301587] LustreError: 26002:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(938504) req@ffff9c145a8b4050 x1655452889579072/t0(0) o4->a1725fdf-452e-7b6d-04bc-4e663aa1a39c@352@gni5:374/0 lens 608/448 e 0 to 0 dl 1585761604 ref 1 fl Interpret:/0/0 rc 0/0 [2644808.301589] LustreError: 26002:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 4 previous similar messages [2644808.301640] Lustre: f2-OST0038: Bulk IO write error with a1725fdf-452e-7b6d-04bc-4e663aa1a39c (at 352@gni5), client will retry: rc = -110 [2644808.301640] Lustre: Skipped 6 previous similar messages [2644813.361273] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.146@o2ib2 added to recovery queue. Health = 900 [2644813.376032] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 3 previous similar messages [2644814.484524] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ll_ost01_069:24698] [2644814.489511] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ll_ost00_022:24581] [2644814.489550] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.489584] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.489586] CPU: 2 PID: 24581 Comm: ll_ost00_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.489587] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.489588] task: ffff9c148eda9070 ti: ffff9c144c154000 task.ti: ffff9c144c154000 [2644814.489595] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.489596] RSP: 0018:ffff9c144c157b90 EFLAGS: 00000246 [2644814.489597] RAX: 0000000000000000 RBX: ffff9c144c157adc RCX: 0000000000110000 [2644814.489597] RDX: ffff9c159f2db880 RSI: 0000000000310000 RDI: ffff9c2a4a5112dc [2644814.489598] RBP: ffff9c144c157b90 R08: ffff9c159f25b880 R09: 0000000000000000 [2644814.489599] R10: ffff9c0b356f4100 R11: ffff9c13efec8000 R12: 0000000000000002 [2644814.489599] R13: ffff9c27aedf6300 R14: ffff9c144c157bd8 R15: 00000000047e5fff [2644814.489600] FS: 0000000000000000(0000) GS:ffff9c159f240000(0000) knlGS:0000000000000000 [2644814.489601] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.489602] CR2: 00007fbc5eceebc0 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.489602] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.489603] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.489603] PKRU: 00000000 [2644814.489604] Call Trace: [2644814.489612] [] queued_spin_lock_slowpath+0xb/0xf [2644814.489616] [] _raw_spin_lock+0x20/0x30 [2644814.489667] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644814.489682] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644814.489702] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644814.489725] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.489757] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.489783] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.489806] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.489811] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.489832] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.489852] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.489855] [] ? __wake_up+0x13/0x20 [2644814.489875] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.489895] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.489897] [] kthread+0xd1/0xe0 [2644814.489899] [] ? insert_kthread_work+0x40/0x40 [2644814.489902] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.489903] [] ? insert_kthread_work+0x40/0x40 [2644814.489917] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.493502] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_ost01_077:24711] [2644814.493537] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.493570] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.493573] CPU: 3 PID: 24711 Comm: ll_ost01_077 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.493574] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.493574] task: ffff9c152b23e2a0 ti: ffff9c142faec000 task.ti: ffff9c142faec000 [2644814.493580] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.493580] RSP: 0018:ffff9c142faefb68 EFLAGS: 00000246 [2644814.493581] RAX: 0000000000000000 RBX: ffff9c142faefb18 RCX: 0000000000190000 [2644814.493582] RDX: ffff9c2d9d61b880 RSI: 0000000000090001 RDI: ffff9c2a4a5112dc [2644814.493582] RBP: ffff9c142faefb68 R08: ffff9c2d9d65b880 R09: 0000000000000000 [2644814.493583] R10: ffff9c147036d800 R11: ffff9c144c6a9fc0 R12: ffff9c263056c850 [2644814.493583] R13: ffff9c2968916000 R14: 00000000bb09818a R15: 0000000000000000 [2644814.493584] FS: 0000000000000000(0000) GS:ffff9c2d9d640000(0000) knlGS:0000000000000000 [2644814.493585] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.493586] CR2: 000000c0014b1000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.493587] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.493587] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.493587] PKRU: 00000000 [2644814.493588] Call Trace: [2644814.493593] [] queued_spin_lock_slowpath+0xb/0xf [2644814.493597] [] _raw_spin_lock+0x20/0x30 [2644814.493632] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.493647] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.493669] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.493687] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.493708] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.493736] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.493760] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.493783] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.493790] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.493810] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.493830] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.493833] [] ? __wake_up+0x13/0x20 [2644814.493852] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.493872] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.493874] [] kthread+0xd1/0xe0 [2644814.493876] [] ? insert_kthread_work+0x40/0x40 [2644814.493878] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.493879] [] ? insert_kthread_work+0x40/0x40 [2644814.493893] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.500485] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ll_ost01_114:24783] [2644814.500511] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.500535] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.500537] CPU: 5 PID: 24783 Comm: ll_ost01_114 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.500537] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.500538] task: ffff9c143c43e2a0 ti: ffff9c143e56c000 task.ti: ffff9c143e56c000 [2644814.500542] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.500543] RSP: 0018:ffff9c143e56fb68 EFLAGS: 00000246 [2644814.500543] RAX: 0000000000000000 RBX: ffff9c143e56fb18 RCX: 0000000000290000 [2644814.500544] RDX: ffff9c159f3db880 RSI: 0000000000710001 RDI: ffff9c2a4a5112dc [2644814.500545] RBP: ffff9c143e56fb68 R08: ffff9c2d9d69b880 R09: 0000000000000000 [2644814.500545] R10: ffff9c147036d800 R11: ffff9c14498adbc0 R12: ffff9c2b2a7ac050 [2644814.500546] R13: ffff9c1fe1c68c00 R14: 000000004be36980 R15: 0000000000000000 [2644814.500547] FS: 0000000000000000(0000) GS:ffff9c2d9d680000(0000) knlGS:0000000000000000 [2644814.500547] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.500548] CR2: 000000c000801000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.500549] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.500549] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.500549] PKRU: 00000000 [2644814.500550] Call Trace: [2644814.500554] [] queued_spin_lock_slowpath+0xb/0xf [2644814.500556] [] _raw_spin_lock+0x20/0x30 [2644814.500580] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.500595] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.500616] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.500634] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.500655] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.500681] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.500706] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.500730] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.500734] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.500755] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.500776] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.500779] [] ? __wake_up+0x13/0x20 [2644814.500799] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.500820] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.500822] [] kthread+0xd1/0xe0 [2644814.500824] [] ? insert_kthread_work+0x40/0x40 [2644814.500826] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.500827] [] ? insert_kthread_work+0x40/0x40 [2644814.500841] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.505472] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ll_ost00_133:26921] [2644814.505500] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.505523] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.505525] CPU: 6 PID: 26921 Comm: ll_ost00_133 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.505526] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.505527] task: ffff9c148fe98000 ti: ffff9c145f204000 task.ti: ffff9c145f204000 [2644814.505531] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.505532] RSP: 0018:ffff9c145f207b90 EFLAGS: 00000246 [2644814.505532] RAX: 0000000000000000 RBX: ffff9c145f207adc RCX: 0000000000310000 [2644814.505533] RDX: ffff9c159f49b880 RSI: 0000000000a10000 RDI: ffff9c2a4a5112dc [2644814.505533] RBP: ffff9c145f207b90 R08: ffff9c159f2db880 R09: 0000000000000000 [2644814.505534] R10: ffff9c0add953100 R11: ffff9c125b528000 R12: 0000000000000002 [2644814.505535] R13: ffff9c267f3c4ec0 R14: ffff9c145f207bd8 R15: 00000000047e5fff [2644814.505536] FS: 0000000000000000(0000) GS:ffff9c159f2c0000(0000) knlGS:0000000000000000 [2644814.505537] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.505537] CR2: 00007f27649f2bc0 CR3: 000000172e636000 CR4: 00000000007607e0 [2644814.505538] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.505539] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.505539] PKRU: 00000000 [2644814.505540] Call Trace: [2644814.505544] [] queued_spin_lock_slowpath+0xb/0xf [2644814.505546] [] _raw_spin_lock+0x20/0x30 [2644814.505570] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644814.505585] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644814.505603] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644814.505624] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.505651] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.505676] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.505700] [] ? tgt_request_preprocess.isra.31+0x7a0/0x7a0 [ptlrpc] [2644814.505721] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.505741] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.505744] [] ? __wake_up+0x13/0x20 [2644814.505763] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.505784] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.505785] [] kthread+0xd1/0xe0 [2644814.505787] [] ? insert_kthread_work+0x40/0x40 [2644814.505789] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.505791] [] ? insert_kthread_work+0x40/0x40 [2644814.505805] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.509463] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ll_ost01_019:23730] [2644814.509489] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.509511] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.509513] CPU: 7 PID: 23730 Comm: ll_ost01_019 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.509514] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.509514] task: ffff9c1571fc8000 ti: ffff9c1542fe0000 task.ti: ffff9c1542fe0000 [2644814.509544] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x224/0x810 [ptlrpc] [2644814.509545] RSP: 0018:ffff9c1542fe3a38 EFLAGS: 00000206 [2644814.509546] RAX: ffff9c1337abd100 RBX: ffff9c1f9fd07300 RCX: 0000000000000006 [2644814.509546] RDX: ffff9c2319ca9920 RSI: 00000000047e5000 RDI: ffff9c145c700400 [2644814.509547] RBP: ffff9c1542fe3ac0 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2644814.509547] R10: 00000000047e5000 R11: 00000000047e5000 R12: 0000000051a7d901 [2644814.509548] R13: ffff9c1f9fd07300 R14: 0000000000000010 R15: ffffffff94223abd [2644814.509549] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2644814.509550] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.509550] CR2: 00007fcb820849ac CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.509551] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.509552] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.509552] PKRU: 00000000 [2644814.509552] Call Trace: [2644814.509570] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2644814.509586] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2644814.509601] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2644814.509615] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2644814.509630] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644814.509647] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644814.509667] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.509693] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.509717] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.509739] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.509744] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.509764] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.509783] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.509786] [] ? queued_spin_lock_slowpath+0xb/0xf [2644814.509787] [] ? _raw_spin_lock+0x20/0x30 [2644814.509807] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.509826] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.509828] [] kthread+0xd1/0xe0 [2644814.509830] [] ? insert_kthread_work+0x40/0x40 [2644814.509832] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.509833] [] ? insert_kthread_work+0x40/0x40 [2644814.509847] Code: 39 d0 48 8d 42 a0 0f 84 b8 04 00 00 48 8b 75 b8 31 c9 eb 24 0f 1f 00 49 8b bf b8 00 00 00 48 39 b8 b8 00 00 00 74 2f 48 8b 50 60 <49> 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 [2644814.513452] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ll_ost00_015:23813] [2644814.513475] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.513495] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.513497] CPU: 8 PID: 23813 Comm: ll_ost00_015 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.513497] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.513498] task: ffff9c2c81e13150 ti: ffff9c2d0f74c000 task.ti: ffff9c2d0f74c000 [2644814.513501] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.513502] RSP: 0018:ffff9c2d0f74fbc8 EFLAGS: 00000246 [2644814.513502] RAX: 0000000000000000 RBX: ffff9c2d0f74fb68 RCX: 0000000000410000 [2644814.513503] RDX: ffff9c2d9d89b880 RSI: 0000000000a90000 RDI: ffff9c2a4a5112dc [2644814.513504] RBP: ffff9c2d0f74fbc8 R08: ffff9c159f31b880 R09: 0000000000000000 [2644814.513504] R10: ffff9c2c5e1fec00 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644814.513505] R13: ffff9c2c5e1fec00 R14: ffff9c2c5e1fec00 R15: 00000000c5d7ddf4 [2644814.513506] FS: 0000000000000000(0000) GS:ffff9c159f300000(0000) knlGS:0000000000000000 [2644814.513506] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.513507] CR2: 00007f22eee3fc60 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.513508] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.513508] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.513508] PKRU: 00000000 [2644814.513509] Call Trace: [2644814.513512] [] queued_spin_lock_slowpath+0xb/0xf [2644814.513513] [] _raw_spin_lock+0x20/0x30 [2644814.513522] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644814.513541] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644814.513562] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.513588] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.513612] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.513635] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.513639] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.513659] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.513679] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.513681] [] ? __wake_up+0x13/0x20 [2644814.513701] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.513721] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.513722] [] kthread+0xd1/0xe0 [2644814.513724] [] ? insert_kthread_work+0x40/0x40 [2644814.513725] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.513727] [] ? insert_kthread_work+0x40/0x40 [2644814.513741] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.517443] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ll_ost01_040:23880] [2644814.517467] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.517488] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.517490] CPU: 9 PID: 23880 Comm: ll_ost01_040 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.517490] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.517491] task: ffff9c2c947ac1c0 ti: ffff9c152ca4c000 task.ti: ffff9c152ca4c000 [2644814.517494] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.517495] RSP: 0018:ffff9c152ca4fad0 EFLAGS: 00000246 [2644814.517495] RAX: 0000000000000000 RBX: ffff9c2a2a402850 RCX: 0000000000490000 [2644814.517496] RDX: ffff9c2d9d85b880 RSI: 0000000000990001 RDI: ffff9c2a4a5112dc [2644814.517496] RBP: ffff9c152ca4fad0 R08: ffff9c2d9d71b880 R09: 0000000000000000 [2644814.517497] R10: ffff9bff3fc07500 R11: ffff9c146f562280 R12: ffffffffc14b9237 [2644814.517497] R13: ffff9c152ca4fa58 R14: 0000000000000003 R15: 0000000000000003 [2644814.517498] FS: 0000000000000000(0000) GS:ffff9c2d9d700000(0000) knlGS:0000000000000000 [2644814.517499] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.517500] CR2: 00007f2765175000 CR3: 00000013b3136000 CR4: 00000000007607e0 [2644814.517501] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.517501] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.517501] PKRU: 00000000 [2644814.517502] Call Trace: [2644814.517505] [] queued_spin_lock_slowpath+0xb/0xf [2644814.517507] [] _raw_spin_lock+0x20/0x30 [2644814.517517] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2644814.517539] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2644814.517546] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2644814.517551] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2644814.517568] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.517589] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.517615] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.517638] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.517661] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.517664] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.517684] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.517704] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.517707] [] ? __wake_up+0x13/0x20 [2644814.517727] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.517747] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.517748] [] kthread+0xd1/0xe0 [2644814.517750] [] ? insert_kthread_work+0x40/0x40 [2644814.517751] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.517753] [] ? insert_kthread_work+0x40/0x40 [2644814.517767] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.521433] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ll_ost00_141:26929] [2644814.521455] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.521475] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.521477] CPU: 10 PID: 26929 Comm: ll_ost00_141 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.521477] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.521478] task: ffff9c1432fec1c0 ti: ffff9c1460c40000 task.ti: ffff9c1460c40000 [2644814.521481] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2644814.521482] RSP: 0018:ffff9c1460c43b68 EFLAGS: 00000246 [2644814.521482] RAX: 0000000000000000 RBX: ffff9c1460c43b18 RCX: 0000000000510000 [2644814.521483] RDX: ffff9c2d9d7db880 RSI: 0000000000790001 RDI: ffff9c2a4a5112dc [2644814.521484] RBP: ffff9c1460c43b68 R08: ffff9c159f35b880 R09: 0000000000000000 [2644814.521484] R10: ffff9c147036d800 R11: ffff9c146746b040 R12: ffff9c145b02d050 [2644814.521485] R13: ffff9c084797d400 R14: 00000000cd6df874 R15: 0000000000000000 [2644814.521486] FS: 0000000000000000(0000) GS:ffff9c159f340000(0000) knlGS:0000000000000000 [2644814.521486] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.521487] CR2: 000000c0014af000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.521488] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.521488] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.521489] PKRU: 00000000 [2644814.521489] Call Trace: [2644814.521492] [] queued_spin_lock_slowpath+0xb/0xf [2644814.521494] [] _raw_spin_lock+0x20/0x30 [2644814.521509] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.521523] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.521544] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.521561] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.521582] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.521607] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.521631] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.521653] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.521657] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.521677] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.521697] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.521699] [] ? __wake_up+0x13/0x20 [2644814.521718] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.521738] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.521739] [] kthread+0xd1/0xe0 [2644814.521741] [] ? insert_kthread_work+0x40/0x40 [2644814.521742] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.521744] [] ? insert_kthread_work+0x40/0x40 [2644814.521758] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2644814.525423] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ll_ost01_031:23828] [2644814.525446] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.525467] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.525469] CPU: 11 PID: 23828 Comm: ll_ost01_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.525469] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.525470] task: ffff9c14c1aa41c0 ti: ffff9c144ab50000 task.ti: ffff9c144ab50000 [2644814.525473] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.525474] RSP: 0018:ffff9c144ab53b68 EFLAGS: 00000246 [2644814.525474] RAX: 0000000000000000 RBX: ffff9c144ab53b18 RCX: 0000000000590000 [2644814.525475] RDX: ffff9c2d9d69b880 RSI: 0000000000290001 RDI: ffff9c2a4a5112dc [2644814.525475] RBP: ffff9c144ab53b68 R08: ffff9c2d9d75b880 R09: 0000000000000000 [2644814.525476] R10: ffff9c147036d800 R11: ffff9c2d986f98c0 R12: ffff9c284b4d1050 [2644814.525477] R13: ffff9c2b34382c00 R14: 00000000a7d54ce4 R15: 0000000000000000 [2644814.525478] FS: 0000000000000000(0000) GS:ffff9c2d9d740000(0000) knlGS:0000000000000000 [2644814.525478] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.525479] CR2: 000000c001299000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.525479] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.525480] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.525480] PKRU: 00000000 [2644814.525481] Call Trace: [2644814.525484] [] queued_spin_lock_slowpath+0xb/0xf [2644814.525486] [] _raw_spin_lock+0x20/0x30 [2644814.525505] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.525520] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.525541] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.525559] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.525579] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.525604] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.525629] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.525652] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.525656] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.525676] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.525696] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.525698] [] ? __wake_up+0x13/0x20 [2644814.525718] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.525737] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.525739] [] kthread+0xd1/0xe0 [2644814.525741] [] ? insert_kthread_work+0x40/0x40 [2644814.525742] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.525743] [] ? insert_kthread_work+0x40/0x40 [2644814.525758] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.529414] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ll_ost00_113:26901] [2644814.529436] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.529456] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.529458] CPU: 12 PID: 26901 Comm: ll_ost00_113 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.529458] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.529459] task: ffff9c1444630000 ti: ffff9c1532ea8000 task.ti: ffff9c1532ea8000 [2644814.529462] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.529463] RSP: 0018:ffff9c1532eabb68 EFLAGS: 00000246 [2644814.529463] RAX: 0000000000000000 RBX: ffff9c1532eabb18 RCX: 0000000000610000 [2644814.529464] RDX: ffff9c2d9d75b880 RSI: 0000000000590001 RDI: ffff9c2a4a5112dc [2644814.529464] RBP: ffff9c1532eabb68 R08: ffff9c159f39b880 R09: 0000000000000000 [2644814.529465] R10: ffff9c147036d800 R11: ffff9c146eba41c0 R12: ffff9c146aa28850 [2644814.529466] R13: ffff9c05fa2ae800 R14: 00000000f1e5d1a6 R15: 0000000000000000 [2644814.529467] FS: 0000000000000000(0000) GS:ffff9c159f380000(0000) knlGS:0000000000000000 [2644814.529467] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.529468] CR2: 0000555ea95608d0 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.529468] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.529469] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.529469] PKRU: 00000000 [2644814.529470] Call Trace: [2644814.529473] [] queued_spin_lock_slowpath+0xb/0xf [2644814.529474] [] _raw_spin_lock+0x20/0x30 [2644814.529492] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.529507] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.529528] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.529545] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.529565] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.529590] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.529614] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.529637] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.529641] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.529661] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.529681] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.529683] [] ? __wake_up+0x13/0x20 [2644814.529702] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.529722] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.529723] [] kthread+0xd1/0xe0 [2644814.529725] [] ? insert_kthread_work+0x40/0x40 [2644814.529727] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.529728] [] ? insert_kthread_work+0x40/0x40 [2644814.529742] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.533405] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ll_ost01_090:24735] [2644814.533428] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.533449] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.533451] CPU: 13 PID: 24735 Comm: ll_ost01_090 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.533451] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.533452] task: ffff9c1577c620e0 ti: ffff9c1431c70000 task.ti: ffff9c1431c70000 [2644814.533455] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.533456] RSP: 0018:ffff9c1431c73b68 EFLAGS: 00000246 [2644814.533457] RAX: 0000000000000000 RBX: ffff9c1431c73b18 RCX: 0000000000690000 [2644814.533457] RDX: ffff9c159f39b880 RSI: 0000000000610001 RDI: ffff9c2a4a5112dc [2644814.533458] RBP: ffff9c1431c73b68 R08: ffff9c2d9d79b880 R09: 0000000000000000 [2644814.533459] R10: ffff9c147036d800 R11: ffff9c2c589bcfc0 R12: ffff9c2b2a7ad850 [2644814.533459] R13: ffff9c29b53cd800 R14: 00000000d3101182 R15: 0000000000000000 [2644814.533460] FS: 0000000000000000(0000) GS:ffff9c2d9d780000(0000) knlGS:0000000000000000 [2644814.533461] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.533461] CR2: 00007f22eedd64b1 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.533462] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.533463] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.533463] PKRU: 00000000 [2644814.533463] Call Trace: [2644814.533466] [] queued_spin_lock_slowpath+0xb/0xf [2644814.533468] [] _raw_spin_lock+0x20/0x30 [2644814.533491] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.533506] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.533527] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.533544] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.533565] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.533590] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.533614] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.533637] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.533642] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.533662] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.533682] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.533685] [] ? __wake_up+0x13/0x20 [2644814.533704] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.533724] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.533726] [] kthread+0xd1/0xe0 [2644814.533728] [] ? insert_kthread_work+0x40/0x40 [2644814.533729] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.533731] [] ? insert_kthread_work+0x40/0x40 [2644814.533745] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.537394] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [ll_ost00_129:26917] [2644814.537417] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.537437] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.537439] CPU: 14 PID: 26917 Comm: ll_ost00_129 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.537439] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.537440] task: ffff9c145a8062a0 ti: ffff9c1448180000 task.ti: ffff9c1448180000 [2644814.537443] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.537444] RSP: 0018:ffff9c1448183b68 EFLAGS: 00000246 [2644814.537444] RAX: 0000000000000000 RBX: ffff9c1448183b18 RCX: 0000000000710000 [2644814.537445] RDX: ffff9c2d9d65b880 RSI: 0000000000190001 RDI: ffff9c2a4a5112dc [2644814.537445] RBP: ffff9c1448183b68 R08: ffff9c159f3db880 R09: 0000000000000000 [2644814.537446] R10: ffff9c147036d800 R11: ffff9c146eba4b40 R12: ffff9c146aa28050 [2644814.537447] R13: ffff9c1601dadc00 R14: 00000000c21fe3c6 R15: 0000000000000000 [2644814.537448] FS: 0000000000000000(0000) GS:ffff9c159f3c0000(0000) knlGS:0000000000000000 [2644814.537448] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.537449] CR2: 000000c001447010 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.537450] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.537450] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.537451] PKRU: 00000000 [2644814.537451] Call Trace: [2644814.537454] [] queued_spin_lock_slowpath+0xb/0xf [2644814.537456] [] _raw_spin_lock+0x20/0x30 [2644814.537471] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.537485] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.537506] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.537524] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.537544] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.537569] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.537593] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.537615] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.537619] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.537639] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.537659] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.537661] [] ? __wake_up+0x13/0x20 [2644814.537680] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.537700] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.537702] [] kthread+0xd1/0xe0 [2644814.537703] [] ? insert_kthread_work+0x40/0x40 [2644814.537705] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.537706] [] ? insert_kthread_work+0x40/0x40 [2644814.537721] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.542382] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ll_ost01_061:24684] [2644814.542404] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.542425] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.542426] CPU: 15 PID: 24684 Comm: ll_ost01_061 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.542427] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.542428] task: ffff9c152eeba0e0 ti: ffff9c151b204000 task.ti: ffff9c151b204000 [2644814.542430] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.542431] RSP: 0018:ffff9c151b207b90 EFLAGS: 00000246 [2644814.542432] RAX: 0000000000000000 RBX: ffff9c151b207adc RCX: 0000000000790000 [2644814.542432] RDX: ffff9c159f25b880 RSI: 0000000000110000 RDI: ffff9c2a4a5112dc [2644814.542433] RBP: ffff9c151b207b90 R08: ffff9c2d9d7db880 R09: 0000000000000000 [2644814.542434] R10: ffff9c2066209100 R11: ffff9c23f4ee0000 R12: 0000000000000002 [2644814.542434] R13: ffff9c26fb43f740 R14: ffff9c151b207bd8 R15: 00000000047e5fff [2644814.542435] FS: 0000000000000000(0000) GS:ffff9c2d9d7c0000(0000) knlGS:0000000000000000 [2644814.542436] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.542436] CR2: 000000c000be9000 CR3: 00000013b3136000 CR4: 00000000007607e0 [2644814.542437] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.542438] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.542438] PKRU: 00000000 [2644814.542438] Call Trace: [2644814.542441] [] queued_spin_lock_slowpath+0xb/0xf [2644814.542443] [] _raw_spin_lock+0x20/0x30 [2644814.542462] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644814.542478] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644814.542495] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644814.542517] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.542542] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.542566] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.542589] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.542593] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.542613] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.542633] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.542635] [] ? __wake_up+0x13/0x20 [2644814.542654] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.542674] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.542675] [] kthread+0xd1/0xe0 [2644814.542677] [] ? insert_kthread_work+0x40/0x40 [2644814.542679] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.542680] [] ? insert_kthread_work+0x40/0x40 [2644814.542694] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.550362] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ll_ost01_095:24750] [2644814.550384] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.550405] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.550406] CPU: 17 PID: 24750 Comm: ll_ost01_095 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.550407] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.550408] task: ffff9c1439219070 ti: ffff9c143909c000 task.ti: ffff9c143909c000 [2644814.550411] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.550411] RSP: 0018:ffff9c143909fb90 EFLAGS: 00000246 [2644814.550412] RAX: 0000000000000000 RBX: ffff9c143909fadc RCX: 0000000000890000 [2644814.550412] RDX: ffff9c159f31b880 RSI: 0000000000410000 RDI: ffff9c2a4a5112dc [2644814.550413] RBP: ffff9c143909fb90 R08: ffff9c2d9d81b880 R09: 0000000000000000 [2644814.550414] R10: ffff9c1daf956500 R11: ffff9c1d46e00000 R12: 0000000000000002 [2644814.550414] R13: ffff9c26bf809b00 R14: ffff9c143909fbd8 R15: 00000000047e5fff [2644814.550415] FS: 0000000000000000(0000) GS:ffff9c2d9d800000(0000) knlGS:0000000000000000 [2644814.550416] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.550416] CR2: 000000c001513010 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.550417] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.550418] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.550418] PKRU: 00000000 [2644814.550418] Call Trace: [2644814.550421] [] queued_spin_lock_slowpath+0xb/0xf [2644814.550423] [] _raw_spin_lock+0x20/0x30 [2644814.550443] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644814.550458] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644814.550476] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2644814.550496] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.550522] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.550547] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.550570] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.550574] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.550594] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.550614] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.550616] [] ? wake_up_state+0x20/0x20 [2644814.550636] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.550656] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.550657] [] kthread+0xd1/0xe0 [2644814.550659] [] ? insert_kthread_work+0x40/0x40 [2644814.550660] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.550662] [] ? insert_kthread_work+0x40/0x40 [2644814.550676] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.558343] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ll_ost01_017:23725] [2644814.558364] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.558385] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.558386] CPU: 19 PID: 23725 Comm: ll_ost01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.558387] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.558387] task: ffff9c148e91e2a0 ti: ffff9c152f354000 task.ti: ffff9c152f354000 [2644814.558390] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.558391] RSP: 0018:ffff9c152f357b68 EFLAGS: 00000246 [2644814.558391] RAX: 0000000000000000 RBX: ffff9c152f357b18 RCX: 0000000000990000 [2644814.558392] RDX: ffff9c2d9d79b880 RSI: 0000000000690001 RDI: ffff9c2a4a5112dc [2644814.558393] RBP: ffff9c152f357b68 R08: ffff9c2d9d85b880 R09: 0000000000000000 [2644814.558393] R10: ffff9c147036d800 R11: ffff9c2d7e734d80 R12: ffff9c1aaabb4050 [2644814.558394] R13: ffff9c218daccc00 R14: 00000000e79d483e R15: 0000000000000000 [2644814.558395] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2644814.558395] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.558396] CR2: 000000c001307010 CR3: 000000172e636000 CR4: 00000000007607e0 [2644814.558397] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.558397] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.558397] PKRU: 00000000 [2644814.558398] Call Trace: [2644814.558401] [] queued_spin_lock_slowpath+0xb/0xf [2644814.558403] [] _raw_spin_lock+0x20/0x30 [2644814.558421] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644814.558436] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644814.558457] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644814.558475] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644814.558496] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.558521] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.558547] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.558569] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.558573] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.558593] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.558613] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.558615] [] ? __wake_up+0x13/0x20 [2644814.558635] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.558655] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.558657] [] kthread+0xd1/0xe0 [2644814.558658] [] ? insert_kthread_work+0x40/0x40 [2644814.558660] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.558661] [] ? insert_kthread_work+0x40/0x40 [2644814.558675] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.562337] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_cn00_060:73480] [2644814.562359] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.562379] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.562381] CPU: 20 PID: 73480 Comm: ldlm_cn00_060 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.562382] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.562382] task: ffff9c289a4c5230 ti: ffff9c152d6c4000 task.ti: ffff9c152d6c4000 [2644814.562385] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644814.562386] RSP: 0018:ffff9c152d6c7c18 EFLAGS: 00000246 [2644814.562387] RAX: 0000000000000000 RBX: ffff9c152d6c7bc0 RCX: 0000000000a10000 [2644814.562387] RDX: ffff9c2d9d81b880 RSI: 0000000000890001 RDI: ffff9c2a4a5112dc [2644814.562388] RBP: ffff9c152d6c7c18 R08: ffff9c159f49b880 R09: 0000000000000000 [2644814.562388] R10: 0000000000000001 R11: ffff9c06d6ce2100 R12: ffff9c06d6ce2100 [2644814.562389] R13: 0000000000000292 R14: 0000000000000000 R15: ffff9c06d6ce2100 [2644814.562390] FS: 0000000000000000(0000) GS:ffff9c159f480000(0000) knlGS:0000000000000000 [2644814.562390] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.562391] CR2: 000000c0008e3000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.562392] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.562392] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.562392] PKRU: 00000000 [2644814.562393] Call Trace: [2644814.562396] [] queued_spin_lock_slowpath+0xb/0xf [2644814.562397] [] _raw_spin_lock+0x20/0x30 [2644814.562414] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2644814.562428] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2644814.562446] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2644814.562464] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2644814.562480] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2644814.562501] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.562521] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.562523] [] ? __wake_up+0x13/0x20 [2644814.562543] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.562563] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.562564] [] kthread+0xd1/0xe0 [2644814.562566] [] ? insert_kthread_work+0x40/0x40 [2644814.562567] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.562569] [] ? insert_kthread_work+0x40/0x40 [2644814.562583] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644814.566324] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ll_ost01_025:23789] [2644814.566347] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644814.566367] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644814.566369] CPU: 21 PID: 23789 Comm: ll_ost01_025 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644814.566369] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644814.566370] task: ffff9c165a9b8000 ti: ffff9c2d9b00c000 task.ti: ffff9c2d9b00c000 [2644814.566373] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2644814.566373] RSP: 0018:ffff9c2d9b00fbc8 EFLAGS: 00000202 [2644814.566374] RAX: 0000000000000001 RBX: ffff9c2d9b00fb68 RCX: 0000000000a90000 [2644814.566375] RDX: 0000000000b90001 RSI: 0000000000390000 RDI: ffff9c2a4a5112dc [2644814.566375] RBP: ffff9c2d9b00fbc8 R08: ffff9c2d9d89b880 R09: ffff9c159f31b880 [2644814.566376] R10: ffff9c2bfbd0c800 R11: 0000000000000001 R12: ffff9c2a4a5112c0 [2644814.566376] R13: ffff9c2bfbd0c800 R14: ffff9c2bfbd0c800 R15: 00000000aa11501e [2644814.566377] FS: 0000000000000000(0000) GS:ffff9c2d9d880000(0000) knlGS:0000000000000000 [2644814.566378] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644814.566378] CR2: 000000c0009a1000 CR3: 000000190da10000 CR4: 00000000007607e0 [2644814.566379] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644814.566380] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644814.566380] PKRU: 00000000 [2644814.566380] Call Trace: [2644814.566384] [] queued_spin_lock_slowpath+0xb/0xf [2644814.566386] [] _raw_spin_lock+0x20/0x30 [2644814.566392] [] ofd_lvbo_fill+0x42/0xa0 [ofd] [2644814.566413] [] ldlm_handle_enqueue0+0x5c8/0x15f0 [ptlrpc] [2644814.566434] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644814.566460] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644814.566484] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644814.566507] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644814.566511] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644814.566531] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644814.566551] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644814.566553] [] ? __wake_up+0x13/0x20 [2644814.566573] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644814.566592] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644814.566594] [] kthread+0xd1/0xe0 [2644814.566596] [] ? insert_kthread_work+0x40/0x40 [2644814.566597] [] ret_from_fork_nospec_begin+0x7/0x21 [2644814.566598] [] ? insert_kthread_work+0x40/0x40 [2644814.566613] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2644816.813866] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2644816.813867] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 10 previous similar messages [2644816.813870] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.33.4@o2ib2 (20): c: 7, oc: 0, rc: 8 [2644816.813872] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 10 previous similar messages [2644822.246862] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644822.324171] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644822.397253] CPU: 1 PID: 24698 Comm: ll_ost01_069 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644822.410731] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644822.419188] task: ffff9c152a348000 ti: ffff9c142e43c000 task.ti: ffff9c142e43c000 [2644822.427555] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644822.438381] RSP: 0018:ffff9c142e43fb68 EFLAGS: 00000246 [2644822.444596] RAX: 0000000000000000 RBX: ffff9c142e43fb18 RCX: 0000000000090000 [2644822.452646] RDX: ffff9c159f35b880 RSI: 0000000000510001 RDI: ffff9c2a4a5112dc [2644822.460694] RBP: ffff9c142e43fb68 R08: ffff9c2d9d61b880 R09: 0000000000000000 [2644822.468743] R10: ffff9c147036d800 R11: ffff9c144c6b0d00 R12: ffff9c1e79ef2050 [2644822.476793] R13: ffff9c2a286a2400 R14: 000000004ceb9c4f R15: 0000000000000000 [2644822.484850] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2644822.493858] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644822.500535] CR2: 0000555ea95ce1e0 CR3: 000000172e636000 CR4: 00000000007607e0 [2644822.508600] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644822.516658] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644822.524715] PKRU: 00000000 [2644822.528354] Call Trace: [2644822.531730] [] queued_spin_lock_slowpath+0xb/0xf [2644822.538922] [] _raw_spin_lock+0x20/0x30 [2644822.545352] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2644822.552814] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2644822.554886] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_ost01_122:24802] [2644822.554907] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2644822.554927] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2644822.554929] CPU: 23 PID: 24802 Comm: ll_ost01_122 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2644822.554929] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2644822.554930] task: ffff9c14540dc1c0 ti: ffff9c1454908000 task.ti: ffff9c1454908000 [2644822.554933] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2644822.554934] RSP: 0018:ffff9c145490bad0 EFLAGS: 00000246 [2644822.554934] RAX: 0000000000000000 RBX: ffff9c279757e850 RCX: 0000000000b90000 [2644822.554935] RDX: ffff9c2d9d71b880 RSI: 0000000000490001 RDI: ffff9c2a4a5112dc [2644822.554936] RBP: ffff9c145490bad0 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2644822.554936] R10: ffff9bff3fc07500 R11: ffff9c144a517f40 R12: ffffffffc14b9237 [2644822.554937] R13: ffff9c145490ba58 R14: 0000000000000003 R15: 0000000000000003 [2644822.554938] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2644822.554938] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2644822.554939] CR2: 00007f77d9493000 CR3: 0000002fa937e000 CR4: 00000000007607e0 [2644822.554940] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2644822.554940] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2644822.554940] PKRU: 00000000 [2644822.554941] Call Trace: [2644822.554944] [] queued_spin_lock_slowpath+0xb/0xf [2644822.554946] [] _raw_spin_lock+0x20/0x30 [2644822.554955] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2644822.554971] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2644822.554977] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2644822.554982] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2644822.555001] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644822.555021] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644822.555047] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644822.555071] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644822.555093] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644822.555097] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644822.555117] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644822.555137] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644822.555139] [] ? __wake_up+0x13/0x20 [2644822.555159] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644822.555178] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644822.555180] [] kthread+0xd1/0xe0 [2644822.555181] [] ? insert_kthread_work+0x40/0x40 [2644822.555183] [] ret_from_fork_nospec_begin+0x7/0x21 [2644822.555185] [] ? insert_kthread_work+0x40/0x40 [2644822.555199] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644823.028841] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2644823.036335] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2644823.044169] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2644823.052431] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2644823.059286] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2644823.066911] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2644823.075198] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2644823.082973] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2644823.091339] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2644823.098816] [] ? __wake_up+0x13/0x20 [2644823.104749] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2644823.111710] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2644823.119771] [] kthread+0xd1/0xe0 [2644823.125323] [] ? insert_kthread_work+0x40/0x40 [2644823.132083] [] ret_from_fork_nospec_begin+0x7/0x21 [2644823.139189] [] ? insert_kthread_work+0x40/0x40 [2644823.145950] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2644824.543305] Lustre: 24753:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1585761562/real 1585761576] req@ffff9c092a144800 x1660035268938048/t0(0) o104->f2-OST0038@10.10.33.4@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1585761880 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [2644824.563112] LustreError: 25963:0:(ldlm_lib.c:3277:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff9c1c96f28850 x1655452665911216/t0(0) o4->32ad47db-af7e-bb55-a5f3-463b5c6d6e94@212@gni5:401/0 lens 608/448 e 0 to 0 dl 1585761631 ref 1 fl Interpret:/0/0 rc 0/0 [2644824.563190] Lustre: f2-OST0014: Bulk IO write error with 32ad47db-af7e-bb55-a5f3-463b5c6d6e94 (at 212@gni5), client will retry: rc = -110 [2644824.563191] Lustre: Skipped 16 previous similar messages [2644824.617999] Lustre: 24753:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 17 previous similar messages [2644856.114059] Lustre: f2-OST0038: deleting orphan objects from 0x0:6898803 to 0x0:6898826 [2644860.921800] Lustre: f2-OST0038: deleting orphan objects from 0x940000402:3496069 to 0x940000402:3496097 [2644860.941412] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000402:3497829 to 0x7c0000402:3497857 [2644864.748099] Lustre: f2-OST0014: Client f2-MDT0001-mdtlov_UUID (at 10.10.33.7@o2ib2) reconnecting [2644864.748200] Lustre: f2-OST0038: deleting orphan objects from 0x940000401:4564586 to 0x940000401:4564609 [2644864.768062] Lustre: Skipped 54 previous similar messages [2644864.774313] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000401:4562762 to 0x7c0000401:4562785 [2645363.225246] Lustre: f2-OST0014: Client 3964c497-b39d-8564-eb85-54a400de46f9 (at 10.10.33.4@o2ib2) reconnecting [2645363.236154] Lustre: Skipped 182 previous similar messages [2645363.242474] Lustre: f2-OST0014: Connection restored to d28ef103-ecee-6bbc-6acb-e927a712e686 (at 10.10.33.4@o2ib2) [2645363.253602] Lustre: Skipped 200 previous similar messages [2652200.600076] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [ldlm_cn00_035:56823] [2652200.604065] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ll_ost01_045:24586] [2652200.604099] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2652200.604132] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2652200.604135] CPU: 23 PID: 24586 Comm: ll_ost01_045 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2652200.604136] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2652200.604137] task: ffff9c152c3ee2a0 ti: ffff9c1451b30000 task.ti: ffff9c1451b30000 [2652200.604142] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2652200.604143] RSP: 0018:ffff9c1451b33ad0 EFLAGS: 00000202 [2652200.604143] RAX: 0000000000000001 RBX: ffff9c2c771b4850 RCX: 0000000000000001 [2652200.604144] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2652200.604144] RBP: ffff9c1451b33ad0 R08: 0000000000000101 R09: 0000000000000158 [2652200.604145] R10: ffff9bff3fc07500 R11: ffff9c144aace380 R12: ffffffffc14b9237 [2652200.604145] R13: ffff9c1451b33a58 R14: 0000000000000003 R15: 0000000000000003 [2652200.604146] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2652200.604147] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2652200.604148] CR2: 000000c001305000 CR3: 0000002f0b7ba000 CR4: 00000000007607e0 [2652200.604149] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2652200.604149] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2652200.604149] PKRU: 00000000 [2652200.604150] Call Trace: [2652200.604156] [] queued_spin_lock_slowpath+0xb/0xf [2652200.604159] [] _raw_spin_lock+0x20/0x30 [2652200.604171] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2652200.604204] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2652200.604214] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2652200.604219] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2652200.604239] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2652200.604260] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2652200.604289] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2652200.604314] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2652200.604336] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2652200.604340] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2652200.604360] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652200.604380] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652200.604383] [] ? __wake_up+0x13/0x20 [2652200.604402] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652200.604422] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652200.604424] [] kthread+0xd1/0xe0 [2652200.604426] [] ? insert_kthread_work+0x40/0x40 [2652200.604428] [] ret_from_fork_nospec_begin+0x7/0x21 [2652200.604429] [] ? insert_kthread_work+0x40/0x40 [2652200.604443] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2652201.078737] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2652201.156028] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2652201.228922] CPU: 22 PID: 56823 Comm: ldlm_cn00_035 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2652201.242502] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2652201.250924] task: ffff9c2d74e90000 ti: ffff9c2a4a350000 task.ti: ffff9c2a4a350000 [2652201.259259] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2652201.270359] RSP: 0018:ffff9c2a4a353ac0 EFLAGS: 00000206 [2652201.276540] RAX: ffff9c2721b0ad00 RBX: ffff9c2a4a353a70 RCX: 000000000000000d [2652201.284547] RDX: 0000000000000004 RSI: 000000000480e000 RDI: ffff9c1451abb000 [2652201.292552] RBP: ffff9c2a4a353b48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2652201.300576] R10: 000000000480e000 R11: 000000000480e000 R12: ffff9c2cd2bf2d00 [2652201.308598] R13: ffff9c2a4a353b48 R14: ffffbf02b2877540 R15: ffff9c2cd2bf2d00 [2652201.316624] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2652201.325604] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2652201.332245] CR2: 000000c001381010 CR3: 000000172e636000 CR4: 00000000007607e0 [2652201.340278] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2652201.348309] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2652201.356340] PKRU: 00000000 [2652201.359954] Call Trace: [2652201.363341] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2652201.371551] [] ? __slab_free+0x1b0/0x290 [2652201.378039] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2652201.386435] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2652201.394305] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2652201.402524] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2652201.410401] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2652201.417934] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2652201.425723] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2652201.433426] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2652201.441222] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652201.449795] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652201.457489] [] ? __wake_up+0x13/0x20 [2652201.463650] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652201.470829] [] ? finish_task_switch+0xe0/0x1c0 [2652201.477841] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652201.486123] [] kthread+0xd1/0xe0 [2652201.491871] [] ? insert_kthread_work+0x40/0x40 [2652201.498823] [] ret_from_fork_nospec_begin+0x7/0x21 [2652201.506105] [] ? insert_kthread_work+0x40/0x40 [2652201.513033] Code: b8 00 00 00 74 2f 48 8b 50 60 49 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 00 49 39 c7 74 e1 8b 90 98 00 00 00 <44> 85 24 95 00 01 5a c1 75 c1 83 c1 01 48 89 f7 48 8b 90 d0 00 [2652208.519806] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ll_ost01_025:23789] [2652208.528606] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2652208.606114] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2652208.679268] CPU: 7 PID: 23789 Comm: ll_ost01_025 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2652208.692736] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2652208.701183] task: ffff9c165a9b8000 ti: ffff9c2d9b00c000 task.ti: ffff9c2d9b00c000 [2652208.709543] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2652208.720367] RSP: 0018:ffff9c2d9b00fad0 EFLAGS: 00000202 [2652208.726566] RAX: 0000000000000101 RBX: ffff9c2843296050 RCX: 0000000000390000 [2652208.734587] RDX: 0000000000390101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2652208.742613] RBP: ffff9c2d9b00fad0 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2652208.750643] R10: ffff9bff3fc07500 R11: ffff9c2d7e734440 R12: ffffffffc14b9237 [2652208.758676] R13: ffff9c2d9b00fa58 R14: 0000000000000003 R15: 0000000000000003 [2652208.766709] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2652208.775699] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2652208.782347] CR2: 000000c001e63000 CR3: 0000001724304000 CR4: 00000000007607e0 [2652208.790388] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2652208.798420] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2652208.806452] PKRU: 00000000 [2652208.810065] Call Trace: [2652208.813416] [] queued_spin_lock_slowpath+0xb/0xf [2652208.820582] [] _raw_spin_lock+0x20/0x30 [2652208.826977] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2652208.834351] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2652208.841962] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2652208.849997] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2652208.857267] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2652208.865231] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2652208.873632] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2652208.880648] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2652208.888435] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2652208.896899] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2652208.904870] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652208.913445] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652208.921121] [] ? __wake_up+0x13/0x20 [2652208.927244] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652208.934402] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652208.942659] [] kthread+0xd1/0xe0 [2652208.948388] [] ? insert_kthread_work+0x40/0x40 [2652208.955316] [] ret_from_fork_nospec_begin+0x7/0x21 [2652208.962578] [] ? insert_kthread_work+0x40/0x40 [2652208.969475] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2652228.532117] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_cn00_035:56823] [2652228.536106] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_ost01_045:24586] [2652228.536142] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2652228.536176] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2652228.536178] CPU: 23 PID: 24586 Comm: ll_ost01_045 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2652228.536179] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2652228.536180] task: ffff9c152c3ee2a0 ti: ffff9c1451b30000 task.ti: ffff9c1451b30000 [2652228.536184] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2652228.536185] RSP: 0018:ffff9c1451b33ad0 EFLAGS: 00000202 [2652228.536185] RAX: 0000000000000001 RBX: ffff9c2c771b4850 RCX: 0000000000000001 [2652228.536186] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2652228.536187] RBP: ffff9c1451b33ad0 R08: 0000000000000101 R09: 0000000000000158 [2652228.536187] R10: ffff9bff3fc07500 R11: ffff9c144aace380 R12: ffffffffc14b9237 [2652228.536188] R13: ffff9c1451b33a58 R14: 0000000000000003 R15: 0000000000000003 [2652228.536189] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2652228.536189] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2652228.536190] CR2: 000000c001305000 CR3: 000000190da10000 CR4: 00000000007607e0 [2652228.536191] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2652228.536191] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2652228.536192] PKRU: 00000000 [2652228.536192] Call Trace: [2652228.536197] [] queued_spin_lock_slowpath+0xb/0xf [2652228.536200] [] _raw_spin_lock+0x20/0x30 [2652228.536212] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2652228.536245] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2652228.536254] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2652228.536259] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2652228.536278] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2652228.536299] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2652228.536326] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2652228.536351] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2652228.536373] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2652228.536377] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2652228.536397] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652228.536417] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652228.536420] [] ? __wake_up+0x13/0x20 [2652228.536439] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652228.536459] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652228.536461] [] kthread+0xd1/0xe0 [2652228.536463] [] ? insert_kthread_work+0x40/0x40 [2652228.536465] [] ret_from_fork_nospec_begin+0x7/0x21 [2652228.536466] [] ? insert_kthread_work+0x40/0x40 [2652228.536480] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2652229.006114] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2652229.083311] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2652229.156249] CPU: 22 PID: 56823 Comm: ldlm_cn00_035 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2652229.169836] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2652229.178259] task: ffff9c2d74e90000 ti: ffff9c2a4a350000 task.ti: ffff9c2a4a350000 [2652229.186592] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x224/0x810 [ptlrpc] [2652229.197695] RSP: 0018:ffff9c2a4a353ac0 EFLAGS: 00000206 [2652229.203876] RAX: ffff9c26eb7b7980 RBX: ffff9c2a4a353a70 RCX: 0000000000000029 [2652229.211880] RDX: ffff9c1c1c4ecaa0 RSI: 0000000000007000 RDI: ffff9c07af2d2c00 [2652229.219887] RBP: ffff9c2a4a353b48 R08: ffff9c2a4a5112f0 R09: 0000000004796fff [2652229.227902] R10: 0000000004796000 R11: 0000000000007000 R12: ffff9c2356e38c00 [2652229.235916] R13: ffff9c2a4a353b48 R14: 0000000000000010 R15: ffff9c2356e38c00 [2652229.243930] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2652229.252904] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2652229.259544] CR2: 000000c001381010 CR3: 000000190da10000 CR4: 00000000007607e0 [2652229.267578] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2652229.275610] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2652229.283639] PKRU: 00000000 [2652229.287253] Call Trace: [2652229.290639] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2652229.298875] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2652229.307284] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2652229.315154] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2652229.323384] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2652229.331260] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2652229.338794] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2652229.346575] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2652229.354266] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2652229.362054] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652229.370619] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652229.378306] [] ? __wake_up+0x13/0x20 [2652229.384457] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652229.391626] [] ? finish_task_switch+0xe0/0x1c0 [2652229.398642] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652229.406922] [] kthread+0xd1/0xe0 [2652229.412678] [] ? insert_kthread_work+0x40/0x40 [2652229.419640] [] ret_from_fork_nospec_begin+0x7/0x21 [2652229.426926] [] ? insert_kthread_work+0x40/0x40 [2652229.433851] Code: 39 d0 48 8d 42 a0 0f 84 b8 04 00 00 48 8b 75 b8 31 c9 eb 24 0f 1f 00 49 8b bf b8 00 00 00 48 39 b8 b8 00 00 00 74 2f 48 8b 50 60 <49> 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 [2652235.979001] INFO: rcu_sched self-detected stall on CPU { 22} (t=60000 jiffies g=208923542 c=208923541 q=47290) [2652235.990068] Task dump for CPU 22: [2652235.994245] ldlm_cn00_035 R running task 0 56823 2 0x00000008 [2652236.002154] Call Trace: [2652236.005414] [] sched_show_task+0xa8/0x110 [2652236.012496] [] dump_cpu_task+0x39/0x70 [2652236.018679] [] rcu_dump_cpu_stacks+0x90/0xd0 [2652236.025362] [] rcu_check_callbacks+0x442/0x730 [2652236.032202] [] ? tick_sched_do_timer+0x50/0x50 [2652236.039032] [] update_process_times+0x46/0x80 [2652236.045758] [] tick_sched_handle+0x30/0x70 [2652236.052225] [] tick_sched_timer+0x39/0x80 [2652236.058598] [] __hrtimer_run_queues+0x10e/0x270 [2652236.065479] [] hrtimer_interrupt+0xaf/0x1d0 [2652236.071996] [] local_apic_timer_interrupt+0x3b/0x60 [2652236.079207] [] smp_apic_timer_interrupt+0x43/0x60 [2652236.086236] [] apic_timer_interrupt+0x16a/0x170 [2652236.093091] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2652236.102143] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2652236.110166] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2652236.118352] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2652236.126005] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2652236.134008] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2652236.141669] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2652236.148985] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2652236.156559] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2652236.164054] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2652236.171634] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652236.179991] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652236.187460] [] ? __wake_up+0x13/0x20 [2652236.193391] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652236.200341] [] ? finish_task_switch+0xe0/0x1c0 [2652236.207139] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652236.215211] [] kthread+0xd1/0xe0 [2652236.220780] [] ? insert_kthread_work+0x40/0x40 [2652236.227557] [] ret_from_fork_nospec_begin+0x7/0x21 [2652236.234684] [] ? insert_kthread_work+0x40/0x40 [2652236.451847] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ll_ost01_025:23789] [2652236.460513] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2652236.537537] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2652236.610380] CPU: 7 PID: 23789 Comm: ll_ost01_025 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2652236.623769] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2652236.632210] task: ffff9c165a9b8000 ti: ffff9c2d9b00c000 task.ti: ffff9c2d9b00c000 [2652236.640559] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2652236.651377] RSP: 0018:ffff9c2d9b00fad0 EFLAGS: 00000202 [2652236.657583] RAX: 0000000000000101 RBX: ffff9c2843296050 RCX: 0000000000390000 [2652236.665589] RDX: 0000000000390101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2652236.673596] RBP: ffff9c2d9b00fad0 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2652236.681609] R10: ffff9bff3fc07500 R11: ffff9c2d7e734440 R12: ffffffffc14b9237 [2652236.689625] R13: ffff9c2d9b00fa58 R14: 0000000000000003 R15: 0000000000000003 [2652236.697641] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2652236.706612] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2652236.713251] CR2: 000000c001e63000 CR3: 000000190da10000 CR4: 00000000007607e0 [2652236.721275] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2652236.729283] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2652236.737286] PKRU: 00000000 [2652236.740900] Call Trace: [2652236.744234] [] queued_spin_lock_slowpath+0xb/0xf [2652236.751373] [] _raw_spin_lock+0x20/0x30 [2652236.757756] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2652236.765104] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2652236.772688] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2652236.780719] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2652236.787984] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2652236.795927] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2652236.804305] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2652236.811293] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2652236.819064] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2652236.827509] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2652236.835445] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2652236.843997] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2652236.851662] [] ? __wake_up+0x13/0x20 [2652236.857785] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2652236.864945] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2652236.873199] [] kthread+0xd1/0xe0 [2652236.878931] [] ? insert_kthread_work+0x40/0x40 [2652236.885858] [] ret_from_fork_nospec_begin+0x7/0x21 [2652236.893117] [] ? insert_kthread_work+0x40/0x40 [2652236.900017] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2654430.972788] Lustre: f2-OST0014: Connection restored to 03f1986f-51d0-7d18-a7f6-f005c4484f71 (at 10.10.33.26@o2ib2) [2654430.984001] Lustre: Skipped 18 previous similar messages [2654430.985048] Lustre: Mounted f2-client [2654435.260189] Lustre: Unmounted f2-client [2654738.339668] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ll_ost01_040:23880] [2654738.340665] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ldlm_cn00_000:22695] [2654738.340704] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.340738] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.340741] CPU: 0 PID: 22695 Comm: ldlm_cn00_000 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.340742] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.340743] task: ffff9c2c92ef62a0 ti: ffff9c2c93298000 task.ti: ffff9c2c93298000 [2654738.340796] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2654738.340797] RSP: 0018:ffff9c2c9329bac0 EFLAGS: 00000287 [2654738.340798] RAX: ffff9c2a3505cc80 RBX: ffff9c2c9329ba70 RCX: 0000000000000011 [2654738.340799] RDX: 0000000000000004 RSI: 0000000002e5e000 RDI: ffff9c148edf1800 [2654738.340799] RBP: ffff9c2c9329bb48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2654738.340800] R10: 00000000047e5000 R11: 0000000002e5e000 R12: ffff9c2beb652c00 [2654738.340800] R13: ffffffffc149ede0 R14: ffff9c2c9329ba8c R15: ffff9c2beb652c00 [2654738.340801] FS: 0000000000000000(0000) GS:ffff9c159f200000(0000) knlGS:0000000000000000 [2654738.340802] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.340803] CR2: 000000c00226b000 CR3: 000000172e636000 CR4: 00000000007607f0 [2654738.340803] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.340804] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.340804] PKRU: 00000000 [2654738.340805] Call Trace: [2654738.340824] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2654738.340828] [] ? __slab_free+0x1b0/0x290 [2654738.340844] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2654738.340859] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2654738.340867] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2654738.340882] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2654738.340897] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2654738.340915] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2654738.340932] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2654738.340948] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2654738.340971] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.340992] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.340996] [] ? __wake_up+0x13/0x20 [2654738.341016] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.341019] [] ? __schedule+0x402/0x840 [2654738.341039] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.341041] [] kthread+0xd1/0xe0 [2654738.341043] [] ? insert_kthread_work+0x40/0x40 [2654738.341046] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.341047] [] ? insert_kthread_work+0x40/0x40 [2654738.341061] Code: b8 00 00 00 74 2f 48 8b 50 60 49 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 00 49 39 c7 74 e1 8b 90 98 00 00 00 <44> 85 24 95 00 01 5a c1 75 c1 83 c1 01 48 89 f7 48 8b 90 d0 00 [2654738.344655] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ll_ost00_096:24791] [2654738.344680] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.344703] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.344705] CPU: 2 PID: 24791 Comm: ll_ost00_096 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.344706] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.344707] task: ffff9c1442810000 ti: ffff9c14428a8000 task.ti: ffff9c14428a8000 [2654738.344712] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2654738.344713] RSP: 0018:ffff9c14428abb68 EFLAGS: 00000202 [2654738.344714] RAX: 0000000000000001 RBX: ffff9c14428abaf0 RCX: 0000000000000001 [2654738.344714] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2654738.344715] RBP: ffff9c14428abb68 R08: 0000000000000101 R09: 0000000000000000 [2654738.344715] R10: ffff9c147036d800 R11: ffff9c144a9ec2c0 R12: 0000000000000010 [2654738.344716] R13: ffffffffc14f15c9 R14: ffffffffffffff45 R15: ffff9c2cc63a0000 [2654738.344717] FS: 0000000000000000(0000) GS:ffff9c159f240000(0000) knlGS:0000000000000000 [2654738.344717] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.344718] CR2: 000000c001401000 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.344719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.344719] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.344720] PKRU: 00000000 [2654738.344720] Call Trace: [2654738.344726] [] queued_spin_lock_slowpath+0xb/0xf [2654738.344729] [] _raw_spin_lock+0x20/0x30 [2654738.344757] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.344771] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.344794] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.344812] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.344832] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.344863] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.344888] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.344910] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.344915] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.344935] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.344955] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.344957] [] ? __wake_up+0x13/0x20 [2654738.344977] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.344996] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.344998] [] kthread+0xd1/0xe0 [2654738.344999] [] ? insert_kthread_work+0x40/0x40 [2654738.345001] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.345003] [] ? insert_kthread_work+0x40/0x40 [2654738.345017] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2654738.348645] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_ost01_020:23733] [2654738.348678] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.348708] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.348711] CPU: 3 PID: 23733 Comm: ll_ost01_020 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.348711] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.348713] task: ffff9c154e7ba0e0 ti: ffff9bfddeb98000 task.ti: ffff9bfddeb98000 [2654738.348718] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.348718] RSP: 0018:ffff9bfddeb9bb68 EFLAGS: 00000246 [2654738.348719] RAX: 0000000000000000 RBX: ffff9bfddeb9bb18 RCX: 0000000000190000 [2654738.348720] RDX: ffff9c2d9d89b880 RSI: 0000000000a90101 RDI: ffff9c2a4a5112dc [2654738.348720] RBP: ffff9bfddeb9bb68 R08: ffff9c2d9d65b880 R09: 0000000000000000 [2654738.348721] R10: ffff9c147036d800 R11: ffff9c2c71b9de80 R12: ffff9c24ce83a050 [2654738.348721] R13: ffff9c1466e0f400 R14: 00000000f67f1f72 R15: 0000000000000000 [2654738.348722] FS: 0000000000000000(0000) GS:ffff9c2d9d640000(0000) knlGS:0000000000000000 [2654738.348723] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.348724] CR2: 000000c0008c9000 CR3: 00000017dbfc8000 CR4: 00000000007607e0 [2654738.348724] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.348725] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.348725] PKRU: 00000000 [2654738.348726] Call Trace: [2654738.348731] [] queued_spin_lock_slowpath+0xb/0xf [2654738.348734] [] _raw_spin_lock+0x20/0x30 [2654738.348767] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.348782] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.348805] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.348823] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.348843] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.348873] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.348897] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.348919] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.348926] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.348946] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.348966] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.348969] [] ? __wake_up+0x13/0x20 [2654738.348988] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.349008] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.349010] [] kthread+0xd1/0xe0 [2654738.349012] [] ? insert_kthread_work+0x40/0x40 [2654738.349014] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.349015] [] ? insert_kthread_work+0x40/0x40 [2654738.349030] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.351639] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ll_ost00_007:22883] [2654738.351661] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.351682] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.351684] CPU: 4 PID: 22883 Comm: ll_ost00_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.351684] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.351686] task: ffff9c2d99c041c0 ti: ffff9c2c8bbac000 task.ti: ffff9c2c8bbac000 [2654738.351689] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.351690] RSP: 0018:ffff9c2c8bbafb68 EFLAGS: 00000246 [2654738.351690] RAX: 0000000000000000 RBX: ffff9c2c8bbafb18 RCX: 0000000000210000 [2654738.351691] RDX: ffff9c2d9d75b880 RSI: 0000000000590101 RDI: ffff9c2a4a5112dc [2654738.351691] RBP: ffff9c2c8bbafb68 R08: ffff9c159f29b880 R09: 0000000000000000 [2654738.351692] R10: ffff9c147036d800 R11: ffff9c2d99fc5c00 R12: ffff9c1451a33850 [2654738.351692] R13: ffff9c145b16a000 R14: 0000000026bea78d R15: 0000000000000000 [2654738.351693] FS: 0000000000000000(0000) GS:ffff9c159f280000(0000) knlGS:0000000000000000 [2654738.351694] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.351695] CR2: 00007f76f4e82000 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.351695] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.351696] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.351696] PKRU: 00000000 [2654738.351697] Call Trace: [2654738.351700] [] queued_spin_lock_slowpath+0xb/0xf [2654738.351702] [] _raw_spin_lock+0x20/0x30 [2654738.351719] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.351734] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.351755] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.351773] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.351793] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.351819] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.351844] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.351867] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.351871] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.351892] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.351912] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.351914] [] ? __wake_up+0x13/0x20 [2654738.351934] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.351954] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.351956] [] kthread+0xd1/0xe0 [2654738.351957] [] ? insert_kthread_work+0x40/0x40 [2654738.351959] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.351960] [] ? insert_kthread_work+0x40/0x40 [2654738.351975] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.355628] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ll_ost01_111:24778] [2654738.355653] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.355676] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.355678] CPU: 5 PID: 24778 Comm: ll_ost01_111 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.355679] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.355680] task: ffff9c143c439070 ti: ffff9c143caf0000 task.ti: ffff9c143caf0000 [2654738.355684] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.355684] RSP: 0018:ffff9c143caf3b68 EFLAGS: 00000246 [2654738.355685] RAX: 0000000000000000 RBX: ffff9c143caf3b18 RCX: 0000000000290000 [2654738.355685] RDX: ffff9c2d9d61b880 RSI: 0000000000090101 RDI: ffff9c2a4a5112dc [2654738.355686] RBP: ffff9c143caf3b68 R08: ffff9c2d9d69b880 R09: 0000000000000000 [2654738.355687] R10: ffff9c147036d800 R11: ffff9c14498ad6c0 R12: ffff9c295d0f1050 [2654738.355687] R13: ffff9c2066208000 R14: 000000009c02e002 R15: 0000000000000000 [2654738.355688] FS: 0000000000000000(0000) GS:ffff9c2d9d680000(0000) knlGS:0000000000000000 [2654738.355689] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.355689] CR2: 00007f77d9493000 CR3: 000000172631c000 CR4: 00000000007607e0 [2654738.355690] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.355691] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.355691] PKRU: 00000000 [2654738.355691] Call Trace: [2654738.355695] [] queued_spin_lock_slowpath+0xb/0xf [2654738.355697] [] _raw_spin_lock+0x20/0x30 [2654738.355720] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.355735] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.355756] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.355774] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.355794] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.355820] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.355844] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.355867] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.355871] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.355892] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.355912] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.355914] [] ? __wake_up+0x13/0x20 [2654738.355933] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.355953] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.355955] [] kthread+0xd1/0xe0 [2654738.355957] [] ? insert_kthread_work+0x40/0x40 [2654738.355958] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.355960] [] ? insert_kthread_work+0x40/0x40 [2654738.355974] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.360620] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ll_ost00_077:24749] [2654738.360647] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.360668] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.360670] CPU: 6 PID: 24749 Comm: ll_ost00_077 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.360671] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.360672] task: ffff9c1439218000 ti: ffff9c1438c34000 task.ti: ffff9c1438c34000 [2654738.360674] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.360675] RSP: 0018:ffff9c1438c37b68 EFLAGS: 00000246 [2654738.360676] RAX: 0000000000000000 RBX: ffff9c1438c37b18 RCX: 0000000000310000 [2654738.360677] RDX: ffff9c159f45b880 RSI: 0000000000910101 RDI: ffff9c2a4a5112dc [2654738.360677] RBP: ffff9c1438c37b68 R08: ffff9c159f2db880 R09: 0000000000000000 [2654738.360678] R10: ffff9c147036d800 R11: ffff9c144a9ecc80 R12: ffff9c1451ab3050 [2654738.360678] R13: ffff9c145a932400 R14: 00000000b1495207 R15: 0000000000000000 [2654738.360679] FS: 0000000000000000(0000) GS:ffff9c159f2c0000(0000) knlGS:0000000000000000 [2654738.360680] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.360681] CR2: 000000c001993010 CR3: 0000002ef73b8000 CR4: 00000000007607e0 [2654738.360682] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.360682] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.360683] PKRU: 00000000 [2654738.360683] Call Trace: [2654738.360686] [] queued_spin_lock_slowpath+0xb/0xf [2654738.360688] [] _raw_spin_lock+0x20/0x30 [2654738.360706] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.360721] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.360742] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.360760] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.360781] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.360807] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.360831] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.360854] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.360858] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.360878] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.360898] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.360900] [] ? __wake_up+0x13/0x20 [2654738.360920] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.360939] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.360941] [] kthread+0xd1/0xe0 [2654738.360943] [] ? insert_kthread_work+0x40/0x40 [2654738.360944] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.360945] [] ? insert_kthread_work+0x40/0x40 [2654738.360960] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.364606] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ll_ost01_026:23803] [2654738.364628] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.364648] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.364649] CPU: 7 PID: 23803 Comm: ll_ost01_026 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.364650] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.364651] task: ffff9c2c944ba0e0 ti: ffff9c2cb0f1c000 task.ti: ffff9c2cb0f1c000 [2654738.364654] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.364654] RSP: 0018:ffff9c2cb0f1fb68 EFLAGS: 00000246 [2654738.364655] RAX: 0000000000000000 RBX: ffff9c2cb0f1fb18 RCX: 0000000000390000 [2654738.364655] RDX: ffff9c159f2db880 RSI: 0000000000310101 RDI: ffff9c2a4a5112dc [2654738.364656] RBP: ffff9c2cb0f1fb68 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2654738.364657] R10: ffff9c147036d800 R11: ffff9c2d7e7347c0 R12: ffff9c24ce83b850 [2654738.364657] R13: ffff9c2d32451c00 R14: 00000000d126e9c8 R15: 0000000000000000 [2654738.364658] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2654738.364659] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.364659] CR2: 000000c00227d000 CR3: 00000017dbfc8000 CR4: 00000000007607e0 [2654738.364660] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.364661] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.364661] PKRU: 00000000 [2654738.364661] Call Trace: [2654738.364664] [] queued_spin_lock_slowpath+0xb/0xf [2654738.364666] [] _raw_spin_lock+0x20/0x30 [2654738.364682] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.364696] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.364717] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.364734] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.364755] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.364780] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.364804] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.364827] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.364831] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.364851] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.364870] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.364872] [] ? wake_up_state+0x20/0x20 [2654738.364892] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.364911] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.364913] [] kthread+0xd1/0xe0 [2654738.364914] [] ? insert_kthread_work+0x40/0x40 [2654738.364916] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.364917] [] ? insert_kthread_work+0x40/0x40 [2654738.364931] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.368596] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ll_ost00_050:24694] [2654738.368619] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.368638] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.368640] CPU: 8 PID: 24694 Comm: ll_ost00_050 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.368640] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.368641] task: ffff9c152a34d230 ti: ffff9c1420fbc000 task.ti: ffff9c1420fbc000 [2654738.368644] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.368645] RSP: 0018:ffff9c1420fbfb68 EFLAGS: 00000246 [2654738.368645] RAX: 0000000000000000 RBX: ffff9c1420fbfb18 RCX: 0000000000410000 [2654738.368646] RDX: ffff9c159f4db880 RSI: 0000000000b10101 RDI: ffff9c2a4a5112dc [2654738.368646] RBP: ffff9c1420fbfb68 R08: ffff9c159f31b880 R09: 0000000000000000 [2654738.368647] R10: ffff9c147036d800 R11: ffff9c144c175bc0 R12: ffff9c1451854050 [2654738.368648] R13: ffff9c145039dc00 R14: 000000002ef69a93 R15: 0000000000000000 [2654738.368649] FS: 0000000000000000(0000) GS:ffff9c159f300000(0000) knlGS:0000000000000000 [2654738.368649] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.368650] CR2: 00007f95807e8c60 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.368651] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.368651] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.368652] PKRU: 00000000 [2654738.368652] Call Trace: [2654738.368655] [] queued_spin_lock_slowpath+0xb/0xf [2654738.368657] [] _raw_spin_lock+0x20/0x30 [2654738.368673] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.368688] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.368709] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.368727] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.368748] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.368773] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.368798] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.368820] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.368824] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.368844] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.368864] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.368866] [] ? wake_up_state+0x20/0x20 [2654738.368886] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.368906] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.368908] [] kthread+0xd1/0xe0 [2654738.368909] [] ? insert_kthread_work+0x40/0x40 [2654738.368911] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.368912] [] ? insert_kthread_work+0x40/0x40 [2654738.368926] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.372587] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ll_ost01_018:23726] [2654738.372609] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.372628] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.372630] CPU: 9 PID: 23726 Comm: ll_ost01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.372630] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.372631] task: ffff9c148e919070 ti: ffff9c143347c000 task.ti: ffff9c143347c000 [2654738.372634] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.372635] RSP: 0018:ffff9c143347fad0 EFLAGS: 00000246 [2654738.372635] RAX: 0000000000000000 RBX: ffff9c295d0f4050 RCX: 0000000000490000 [2654738.372636] RDX: ffff9c2d9d79b880 RSI: 0000000000690101 RDI: ffff9c2a4a5112dc [2654738.372636] RBP: ffff9c143347fad0 R08: ffff9c2d9d71b880 R09: 0000000000000000 [2654738.372637] R10: ffff9bff3fc07500 R11: ffff9c2d7e734f40 R12: ffffffffc14b9237 [2654738.372637] R13: ffff9c143347fa58 R14: 0000000000000003 R15: 0000000000000003 [2654738.372638] FS: 0000000000000000(0000) GS:ffff9c2d9d700000(0000) knlGS:0000000000000000 [2654738.372639] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.372640] CR2: 00007fcb82c09000 CR3: 00000017007b8000 CR4: 00000000007607e0 [2654738.372640] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.372641] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.372641] PKRU: 00000000 [2654738.372641] Call Trace: [2654738.372644] [] queued_spin_lock_slowpath+0xb/0xf [2654738.372646] [] _raw_spin_lock+0x20/0x30 [2654738.372658] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2654738.372673] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2654738.372679] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2654738.372684] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2654738.372702] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.372722] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.372748] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.372772] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.372794] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.372798] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.372818] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.372838] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.372840] [] ? __wake_up+0x13/0x20 [2654738.372860] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.372879] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.372881] [] kthread+0xd1/0xe0 [2654738.372882] [] ? insert_kthread_work+0x40/0x40 [2654738.372884] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.372885] [] ? insert_kthread_work+0x40/0x40 [2654738.372900] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.376577] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ll_ost00_113:26901] [2654738.376600] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.376619] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.376621] CPU: 10 PID: 26901 Comm: ll_ost00_113 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.376621] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.376622] task: ffff9c1444630000 ti: ffff9c1532ea8000 task.ti: ffff9c1532ea8000 [2654738.376625] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.376626] RSP: 0018:ffff9c1532eabb68 EFLAGS: 00000246 [2654738.376626] RAX: 0000000000000000 RBX: ffff9c1532eabb18 RCX: 0000000000510000 [2654738.376627] RDX: ffff9c159f39b880 RSI: 0000000000610101 RDI: ffff9c2a4a5112dc [2654738.376627] RBP: ffff9c1532eabb68 R08: ffff9c159f35b880 R09: 0000000000000000 [2654738.376628] R10: ffff9c147036d800 R11: ffff9c146eba41c0 R12: ffff9c12807e9850 [2654738.376628] R13: ffff9bfe26d6ac00 R14: 00000000f1e5d1a6 R15: 0000000000000000 [2654738.376629] FS: 0000000000000000(0000) GS:ffff9c159f340000(0000) knlGS:0000000000000000 [2654738.376630] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.376631] CR2: 000000c000cb1010 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.376631] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.376632] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.376632] PKRU: 00000000 [2654738.376633] Call Trace: [2654738.376636] [] queued_spin_lock_slowpath+0xb/0xf [2654738.376638] [] _raw_spin_lock+0x20/0x30 [2654738.376654] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.376669] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.376690] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.376708] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.376728] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.376754] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.376778] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.376800] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.376804] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.376824] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.376844] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.376846] [] ? __wake_up+0x13/0x20 [2654738.376866] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.376886] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.376887] [] kthread+0xd1/0xe0 [2654738.376889] [] ? insert_kthread_work+0x40/0x40 [2654738.376891] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.376892] [] ? insert_kthread_work+0x40/0x40 [2654738.376906] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.380567] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ll_ost01_089:24734] [2654738.380590] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.380609] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.380611] CPU: 11 PID: 24734 Comm: ll_ost01_089 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.380611] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.380612] task: ffff9c14e470b150 ti: ffff9c14322f8000 task.ti: ffff9c14322f8000 [2654738.380615] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.380615] RSP: 0018:ffff9c14322fbb68 EFLAGS: 00000246 [2654738.380616] RAX: 0000000000000000 RBX: ffff9c14322fbb18 RCX: 0000000000590000 [2654738.380617] RDX: ffff9c159f35b880 RSI: 0000000000510101 RDI: ffff9c2a4a5112dc [2654738.380617] RBP: ffff9c14322fbb68 R08: ffff9c2d9d75b880 R09: 0000000000000000 [2654738.380618] R10: ffff9c147036d800 R11: ffff9c2c589bcf00 R12: ffff9c1c76154850 [2654738.380618] R13: ffff9c1601da8c00 R14: 00000000e2f674e5 R15: 0000000000000000 [2654738.380619] FS: 0000000000000000(0000) GS:ffff9c2d9d740000(0000) knlGS:0000000000000000 [2654738.380620] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.380621] CR2: 000000c000f1b000 CR3: 000000172631c000 CR4: 00000000007607e0 [2654738.380621] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.380622] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.380622] PKRU: 00000000 [2654738.380623] Call Trace: [2654738.380626] [] queued_spin_lock_slowpath+0xb/0xf [2654738.380627] [] _raw_spin_lock+0x20/0x30 [2654738.380645] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.380660] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.380680] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.380698] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.380718] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.380743] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.380767] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.380789] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.380794] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.380814] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.380833] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.380835] [] ? __wake_up+0x13/0x20 [2654738.380855] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.380875] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.380876] [] kthread+0xd1/0xe0 [2654738.380878] [] ? insert_kthread_work+0x40/0x40 [2654738.380879] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.380881] [] ? insert_kthread_work+0x40/0x40 [2654738.380895] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.384558] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ll_ost00_121:26909] [2654738.384581] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.384600] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.384602] CPU: 12 PID: 26909 Comm: ll_ost00_121 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.384603] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.384604] task: ffff9c146007a0e0 ti: ffff9c14447b8000 task.ti: ffff9c14447b8000 [2654738.384607] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.384607] RSP: 0018:ffff9c14447bbb68 EFLAGS: 00000246 [2654738.384608] RAX: 0000000000000000 RBX: ffff9c14447bbb18 RCX: 0000000000610000 [2654738.384609] RDX: ffff9c2d9d7db880 RSI: 0000000000790101 RDI: ffff9c2a4a5112dc [2654738.384609] RBP: ffff9c14447bbb68 R08: ffff9c159f39b880 R09: 0000000000000000 [2654738.384610] R10: ffff9c147036d800 R11: ffff9c146eba4780 R12: ffff9c12c2a46050 [2654738.384610] R13: ffff9c146e285000 R14: 00000000b1575be1 R15: 0000000000000000 [2654738.384611] FS: 0000000000000000(0000) GS:ffff9c159f380000(0000) knlGS:0000000000000000 [2654738.384612] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.384613] CR2: 000000c000879010 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.384613] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.384614] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.384614] PKRU: 00000000 [2654738.384615] Call Trace: [2654738.384618] [] queued_spin_lock_slowpath+0xb/0xf [2654738.384619] [] _raw_spin_lock+0x20/0x30 [2654738.384638] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.384653] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.384673] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.384691] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.384712] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.384737] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.384761] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.384784] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.384788] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.384808] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.384828] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.384830] [] ? __wake_up+0x13/0x20 [2654738.384850] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.384870] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.384871] [] kthread+0xd1/0xe0 [2654738.384873] [] ? insert_kthread_work+0x40/0x40 [2654738.384874] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.384876] [] ? insert_kthread_work+0x40/0x40 [2654738.384890] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.388548] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ll_ost01_106:24769] [2654738.388570] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.388589] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.388591] CPU: 13 PID: 24769 Comm: ll_ost01_106 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.388591] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.388592] task: ffff9c143a75e2a0 ti: ffff9c143aed8000 task.ti: ffff9c143aed8000 [2654738.388595] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.388596] RSP: 0018:ffff9c143aedbb68 EFLAGS: 00000246 [2654738.388596] RAX: 0000000000000000 RBX: ffff9c143aedbb18 RCX: 0000000000690000 [2654738.388597] RDX: ffff9c159f3db880 RSI: 0000000000710101 RDI: ffff9c2a4a5112dc [2654738.388598] RBP: ffff9c143aedbb68 R08: ffff9c2d9d79b880 R09: 0000000000000000 [2654738.388598] R10: ffff9c147036d800 R11: ffff9c14498ad540 R12: ffff9c24ce839850 [2654738.388599] R13: ffff9c280b6ad400 R14: 00000000f8b08fea R15: 0000000000000000 [2654738.388600] FS: 0000000000000000(0000) GS:ffff9c2d9d780000(0000) knlGS:0000000000000000 [2654738.388600] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.388601] CR2: 000000c000edf010 CR3: 0000001724304000 CR4: 00000000007607e0 [2654738.388602] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.388602] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.388602] PKRU: 00000000 [2654738.388603] Call Trace: [2654738.388606] [] queued_spin_lock_slowpath+0xb/0xf [2654738.388607] [] _raw_spin_lock+0x20/0x30 [2654738.388623] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.388637] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.388658] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.388676] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.388696] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.388721] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.388745] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.388768] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.388772] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.388792] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.388812] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.388814] [] ? __wake_up+0x13/0x20 [2654738.388833] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.388853] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.388854] [] kthread+0xd1/0xe0 [2654738.388856] [] ? insert_kthread_work+0x40/0x40 [2654738.388857] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.388859] [] ? insert_kthread_work+0x40/0x40 [2654738.388873] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.392538] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [ll_ost00_075:24746] [2654738.392561] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.392581] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.392583] CPU: 14 PID: 24746 Comm: ll_ost00_075 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.392583] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.392584] task: ffff9c1438eb41c0 ti: ffff9c14392d0000 task.ti: ffff9c14392d0000 [2654738.392587] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.392588] RSP: 0018:ffff9c14392d3b68 EFLAGS: 00000246 [2654738.392588] RAX: 0000000000000000 RBX: ffff9c14392d3b18 RCX: 0000000000710000 [2654738.392589] RDX: ffff9c2d9d85b880 RSI: 0000000000990101 RDI: ffff9c2a4a5112dc [2654738.392589] RBP: ffff9c14392d3b68 R08: ffff9c159f3db880 R09: 0000000000000000 [2654738.392590] R10: ffff9c147036d800 R11: ffff9c144a9ecd40 R12: ffff9c1451851050 [2654738.392591] R13: ffff9c146aa64800 R14: 00000000ca760746 R15: 0000000000000000 [2654738.392592] FS: 0000000000000000(0000) GS:ffff9c159f3c0000(0000) knlGS:0000000000000000 [2654738.392592] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.392593] CR2: 000000c0010e3000 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.392594] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.392594] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.392595] PKRU: 00000000 [2654738.392595] Call Trace: [2654738.392598] [] queued_spin_lock_slowpath+0xb/0xf [2654738.392600] [] _raw_spin_lock+0x20/0x30 [2654738.392617] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.392632] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.392653] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.392670] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.392691] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.392716] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.392740] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.392763] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.392767] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.392787] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.392807] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.392809] [] ? __wake_up+0x13/0x20 [2654738.392829] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.392849] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.392850] [] kthread+0xd1/0xe0 [2654738.392852] [] ? insert_kthread_work+0x40/0x40 [2654738.392853] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.392855] [] ? insert_kthread_work+0x40/0x40 [2654738.392869] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.397526] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ll_ost01_011:23717] [2654738.397549] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.397569] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.397571] CPU: 15 PID: 23717 Comm: ll_ost01_011 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.397571] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.397572] task: ffff9c159676e2a0 ti: ffff9c15287e4000 task.ti: ffff9c15287e4000 [2654738.397575] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.397576] RSP: 0018:ffff9c15287e7b68 EFLAGS: 00000246 [2654738.397576] RAX: 0000000000000000 RBX: ffff9c15287e7b18 RCX: 0000000000790000 [2654738.397577] RDX: ffff9c2d9d71b880 RSI: 0000000000490101 RDI: ffff9c2a4a5112dc [2654738.397578] RBP: ffff9c15287e7b68 R08: ffff9c2d9d7db880 R09: 0000000000000000 [2654738.397578] R10: ffff9c147036d800 R11: ffff9c2d7e734c40 R12: ffff9c276f4ec050 [2654738.397579] R13: ffff9c295a074800 R14: 00000000d8bde239 R15: 0000000000000000 [2654738.397580] FS: 0000000000000000(0000) GS:ffff9c2d9d7c0000(0000) knlGS:0000000000000000 [2654738.397580] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.397581] CR2: 000000c00020c000 CR3: 000000172631c000 CR4: 00000000007607e0 [2654738.397582] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.397582] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.397582] PKRU: 00000000 [2654738.397583] Call Trace: [2654738.397586] [] queued_spin_lock_slowpath+0xb/0xf [2654738.397587] [] _raw_spin_lock+0x20/0x30 [2654738.397603] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.397617] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.397638] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.397655] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.397675] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.397701] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.397725] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.397747] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.397751] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.397771] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.397790] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.397793] [] ? __wake_up+0x13/0x20 [2654738.397812] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.397832] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.397834] [] kthread+0xd1/0xe0 [2654738.397835] [] ? insert_kthread_work+0x40/0x40 [2654738.397837] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.397838] [] ? insert_kthread_work+0x40/0x40 [2654738.397852] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.401516] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ll_ost00_144:26932] [2654738.401538] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.401558] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.401560] CPU: 16 PID: 26932 Comm: ll_ost00_144 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.401560] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.401561] task: ffff9c157e39a0e0 ti: ffff9c145f1d4000 task.ti: ffff9c145f1d4000 [2654738.401564] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.401565] RSP: 0018:ffff9c145f1d7b68 EFLAGS: 00000246 [2654738.401565] RAX: 0000000000000000 RBX: ffff9c145f1d7b18 RCX: 0000000000810000 [2654738.401566] RDX: ffff9c2d9d65b880 RSI: 0000000000190101 RDI: ffff9c2a4a5112dc [2654738.401566] RBP: ffff9c145f1d7b68 R08: ffff9c159f41b880 R09: 0000000000000000 [2654738.401567] R10: ffff9c147036d800 R11: ffff9c146746b2c0 R12: ffff9c12c2a44850 [2654738.401567] R13: ffff9c133e14c800 R14: 0000000063b021e6 R15: 0000000000000000 [2654738.401568] FS: 0000000000000000(0000) GS:ffff9c159f400000(0000) knlGS:0000000000000000 [2654738.401569] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.401570] CR2: 000000c00198b010 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.401570] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.401571] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.401571] PKRU: 00000000 [2654738.401571] Call Trace: [2654738.401574] [] queued_spin_lock_slowpath+0xb/0xf [2654738.401576] [] _raw_spin_lock+0x20/0x30 [2654738.401593] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.401608] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.401629] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.401647] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.401667] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.401692] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.401716] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.401739] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.401743] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.401763] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.401783] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.401785] [] ? __wake_up+0x13/0x20 [2654738.401804] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.401824] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.401826] [] kthread+0xd1/0xe0 [2654738.401827] [] ? insert_kthread_work+0x40/0x40 [2654738.401829] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.401830] [] ? insert_kthread_work+0x40/0x40 [2654738.401844] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.405507] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ll_ost01_120:24798] [2654738.405528] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.405548] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.405550] CPU: 17 PID: 24798 Comm: ll_ost01_120 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.405551] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.405551] task: ffff9c14540d8000 ti: ffff9c144d7c8000 task.ti: ffff9c144d7c8000 [2654738.405554] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.405555] RSP: 0018:ffff9c144d7cbb68 EFLAGS: 00000246 [2654738.405556] RAX: 0000000000000000 RBX: ffff9c144d7cbb18 RCX: 0000000000890000 [2654738.405556] RDX: ffff9c2d9d69b880 RSI: 0000000000290101 RDI: ffff9c2a4a5112dc [2654738.405557] RBP: ffff9c144d7cbb68 R08: ffff9c2d9d81b880 R09: 0000000000000000 [2654738.405557] R10: ffff9c147036d800 R11: ffff9c144a517c80 R12: ffff9c24ce83f850 [2654738.405558] R13: ffff9c106fffd800 R14: 0000000075851d4c R15: 0000000000000000 [2654738.405559] FS: 0000000000000000(0000) GS:ffff9c2d9d800000(0000) knlGS:0000000000000000 [2654738.405559] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.405560] CR2: 00007f556dcf9000 CR3: 00000017bff7c000 CR4: 00000000007607e0 [2654738.405561] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.405561] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.405561] PKRU: 00000000 [2654738.405562] Call Trace: [2654738.405565] [] queued_spin_lock_slowpath+0xb/0xf [2654738.405567] [] _raw_spin_lock+0x20/0x30 [2654738.405582] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.405597] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.405617] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.405635] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.405655] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.405681] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.405705] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.405727] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.405731] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.405751] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.405771] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.405773] [] ? __wake_up+0x13/0x20 [2654738.405792] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.405812] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.405814] [] kthread+0xd1/0xe0 [2654738.405815] [] ? insert_kthread_work+0x40/0x40 [2654738.405817] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.405818] [] ? insert_kthread_work+0x40/0x40 [2654738.405832] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.409497] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ll_ost00_147:26935] [2654738.409519] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.409539] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.409541] CPU: 18 PID: 26935 Comm: ll_ost00_147 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.409542] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.409542] task: ffff9c2c97f7a0e0 ti: ffff9c146163c000 task.ti: ffff9c146163c000 [2654738.409545] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.409546] RSP: 0018:ffff9c146163fb68 EFLAGS: 00000246 [2654738.409546] RAX: 0000000000000000 RBX: ffff9c146163fb18 RCX: 0000000000910000 [2654738.409547] RDX: ffff9c159f31b880 RSI: 0000000000410101 RDI: ffff9c2a4a5112dc [2654738.409547] RBP: ffff9c146163fb68 R08: ffff9c159f45b880 R09: 0000000000000000 [2654738.409548] R10: ffff9c147036d800 R11: ffff9c146746b480 R12: ffff9c1451854850 [2654738.409549] R13: ffff9c19238dec00 R14: 00000000274ceac4 R15: 0000000000000000 [2654738.409550] FS: 0000000000000000(0000) GS:ffff9c159f440000(0000) knlGS:0000000000000000 [2654738.409550] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.409551] CR2: 000000c000e25000 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.409551] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.409552] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.409552] PKRU: 00000000 [2654738.409553] Call Trace: [2654738.409556] [] queued_spin_lock_slowpath+0xb/0xf [2654738.409557] [] _raw_spin_lock+0x20/0x30 [2654738.409574] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.409589] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.409610] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.409628] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.409648] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.409674] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.409698] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.409721] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.409725] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.409745] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.409765] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.409767] [] ? __wake_up+0x13/0x20 [2654738.409786] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.409806] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.409807] [] kthread+0xd1/0xe0 [2654738.409809] [] ? insert_kthread_work+0x40/0x40 [2654738.409811] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.409812] [] ? insert_kthread_work+0x40/0x40 [2654738.409826] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.413487] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ll_ost01_084:24721] [2654738.413509] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.413529] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.413530] CPU: 19 PID: 24721 Comm: ll_ost01_084 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.413531] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.413532] task: ffff9c15983bd230 ti: ffff9c1430eb0000 task.ti: ffff9c1430eb0000 [2654738.413535] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2654738.413535] RSP: 0018:ffff9c1430eb3b68 EFLAGS: 00000246 [2654738.413536] RAX: 0000000000000000 RBX: ffff9c1430eb3b18 RCX: 0000000000990000 [2654738.413537] RDX: ffff9c159f41b880 RSI: 0000000000810101 RDI: ffff9c2a4a5112dc [2654738.413537] RBP: ffff9c1430eb3b68 R08: ffff9c2d9d85b880 R09: 0000000000000000 [2654738.413538] R10: ffff9c147036d800 R11: ffff9c2c589bcb40 R12: ffff9c295d0f2850 [2654738.413538] R13: ffff9c145a880400 R14: 0000000032d06d35 R15: 0000000000000000 [2654738.413539] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2654738.413540] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.413540] CR2: 000000c000ea9000 CR3: 000000172631c000 CR4: 00000000007607e0 [2654738.413541] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.413542] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.413542] PKRU: 00000000 [2654738.413542] Call Trace: [2654738.413545] [] queued_spin_lock_slowpath+0xb/0xf [2654738.413547] [] _raw_spin_lock+0x20/0x30 [2654738.413562] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.413577] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.413598] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.413615] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.413635] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.413660] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.413685] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.413707] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.413711] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.413731] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.413751] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.413753] [] ? __wake_up+0x13/0x20 [2654738.413772] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.413792] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.413793] [] kthread+0xd1/0xe0 [2654738.413795] [] ? insert_kthread_work+0x40/0x40 [2654738.413796] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.413798] [] ? insert_kthread_work+0x40/0x40 [2654738.413812] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2654738.417477] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_ost00_101:24800] [2654738.417499] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.417520] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.417522] CPU: 20 PID: 24800 Comm: ll_ost00_101 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.417522] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.417523] task: ffff9c14540da0e0 ti: ffff9c144d598000 task.ti: ffff9c144d598000 [2654738.417526] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.417526] RSP: 0018:ffff9c144d59bb68 EFLAGS: 00000246 [2654738.417527] RAX: 0000000000000000 RBX: ffff9c144d59bb18 RCX: 0000000000a10000 [2654738.417528] RDX: ffff9c159f29b880 RSI: 0000000000210101 RDI: ffff9c2a4a5112dc [2654738.417528] RBP: ffff9c144d59bb68 R08: ffff9c159f49b880 R09: 0000000000000000 [2654738.417529] R10: ffff9c147036d800 R11: ffff9c144a9e8ec0 R12: ffff9c1451a36850 [2654738.417530] R13: ffff9c2ab62dbc00 R14: 00000000e6395aad R15: 0000000000000000 [2654738.417531] FS: 0000000000000000(0000) GS:ffff9c159f480000(0000) knlGS:0000000000000000 [2654738.417531] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.417532] CR2: 000000c000f4d000 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.417533] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.417533] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.417534] PKRU: 00000000 [2654738.417534] Call Trace: [2654738.417537] [] queued_spin_lock_slowpath+0xb/0xf [2654738.417539] [] _raw_spin_lock+0x20/0x30 [2654738.417555] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.417570] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.417590] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.417608] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.417628] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.417654] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.417678] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.417701] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.417705] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.417726] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.417745] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.417747] [] ? __wake_up+0x13/0x20 [2654738.417767] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.417788] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.417789] [] kthread+0xd1/0xe0 [2654738.417791] [] ? insert_kthread_work+0x40/0x40 [2654738.417792] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.417794] [] ? insert_kthread_work+0x40/0x40 [2654738.417808] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.421468] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ll_ost01_048:24593] [2654738.421491] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.421511] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.421512] CPU: 21 PID: 24593 Comm: ll_ost01_048 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.421513] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.421514] task: ffff9c1597ef3150 ti: ffff9c1456278000 task.ti: ffff9c1456278000 [2654738.421516] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654738.421517] RSP: 0018:ffff9c145627bb68 EFLAGS: 00000246 [2654738.421518] RAX: 0000000000000000 RBX: ffff9c145627bb18 RCX: 0000000000a90000 [2654738.421518] RDX: ffff9c2d9d6db880 RSI: 0000000000390101 RDI: ffff9c2a4a5112dc [2654738.421519] RBP: ffff9c145627bb68 R08: ffff9c2d9d89b880 R09: 0000000000000000 [2654738.421519] R10: ffff9c147036d800 R11: ffff9c144aace640 R12: ffff9c2d0d240050 [2654738.421520] R13: ffff9c21e6b32800 R14: 000000005866dbbf R15: 0000000000000000 [2654738.421521] FS: 0000000000000000(0000) GS:ffff9c2d9d880000(0000) knlGS:0000000000000000 [2654738.421522] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.421522] CR2: 000000c00226d000 CR3: 0000001726826000 CR4: 00000000007607e0 [2654738.421523] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.421523] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.421524] PKRU: 00000000 [2654738.421524] Call Trace: [2654738.421527] [] queued_spin_lock_slowpath+0xb/0xf [2654738.421529] [] _raw_spin_lock+0x20/0x30 [2654738.421544] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.421559] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.421579] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.421597] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.421617] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.421642] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.421666] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.421689] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.421692] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.421713] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.421732] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.421734] [] ? __wake_up+0x13/0x20 [2654738.421754] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.421773] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.421775] [] kthread+0xd1/0xe0 [2654738.421777] [] ? insert_kthread_work+0x40/0x40 [2654738.421778] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.421780] [] ? insert_kthread_work+0x40/0x40 [2654738.421794] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654738.425458] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ll_ost00_051:24695] [2654738.425479] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.425500] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.425502] CPU: 22 PID: 24695 Comm: ll_ost00_051 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.425502] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.425503] task: ffff9c152a349070 ti: ffff9c146248c000 task.ti: ffff9c146248c000 [2654738.425506] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2654738.425506] RSP: 0018:ffff9c146248fb68 EFLAGS: 00000246 [2654738.425507] RAX: 0000000000000000 RBX: ffff9c146248fb18 RCX: 0000000000b10000 [2654738.425507] RDX: ffff9c2d9d81b880 RSI: 0000000000890101 RDI: ffff9c2a4a5112dc [2654738.425508] RBP: ffff9c146248fb68 R08: ffff9c159f4db880 R09: 0000000000000000 [2654738.425509] R10: ffff9c147036d800 R11: ffff9c1440589f80 R12: ffff9c146ac9e050 [2654738.425509] R13: ffff9c27cfff3800 R14: 000000009e0c7042 R15: 0000000000000000 [2654738.425510] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2654738.425511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.425511] CR2: 00007f9d0f438000 CR3: 000000172e636000 CR4: 00000000007607e0 [2654738.425512] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.425513] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.425513] PKRU: 00000000 [2654738.425513] Call Trace: [2654738.425516] [] queued_spin_lock_slowpath+0xb/0xf [2654738.425518] [] _raw_spin_lock+0x20/0x30 [2654738.425535] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.425549] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.425570] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.425588] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.425608] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.425634] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.425658] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.425681] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.425685] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.425705] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.425725] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.425727] [] ? wake_up_state+0x20/0x20 [2654738.425746] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.425766] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.425767] [] kthread+0xd1/0xe0 [2654738.425769] [] ? insert_kthread_work+0x40/0x40 [2654738.425771] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.425772] [] ? insert_kthread_work+0x40/0x40 [2654738.425786] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2654738.429448] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_ost01_050:24597] [2654738.429471] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654738.429490] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654738.429492] CPU: 23 PID: 24597 Comm: ll_ost01_050 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654738.429492] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654738.429493] task: ffff9c1597c05230 ti: ffff9c143e570000 task.ti: ffff9c143e570000 [2654738.429496] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2654738.429497] RSP: 0018:ffff9c143e573b68 EFLAGS: 00000202 [2654738.429497] RAX: 0000000000000101 RBX: ffff9c143e573b18 RCX: 0000000000b90000 [2654738.429498] RDX: 0000000000a10101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2654738.429498] RBP: ffff9c143e573b68 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2654738.429499] R10: ffff9c147036d800 R11: ffff9c144aace900 R12: ffff9c2c793cb850 [2654738.429499] R13: ffff9c2a82b8e000 R14: 00000000ff2d0b38 R15: 0000000000000000 [2654738.429500] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2654738.429501] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654738.429502] CR2: 000000c000eb9010 CR3: 0000002f0b7ba000 CR4: 00000000007607e0 [2654738.429502] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654738.429503] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654738.429503] PKRU: 00000000 [2654738.429503] Call Trace: [2654738.429506] [] queued_spin_lock_slowpath+0xb/0xf [2654738.429508] [] _raw_spin_lock+0x20/0x30 [2654738.429523] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654738.429538] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654738.429558] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654738.429576] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654738.429596] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654738.429621] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654738.429645] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654738.429667] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654738.429671] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654738.429691] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654738.429711] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654738.429713] [] ? __wake_up+0x13/0x20 [2654738.429733] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654738.429753] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654738.429754] [] kthread+0xd1/0xe0 [2654738.429756] [] ? insert_kthread_work+0x40/0x40 [2654738.429757] [] ret_from_fork_nospec_begin+0x7/0x21 [2654738.429759] [] ? insert_kthread_work+0x40/0x40 [2654738.429773] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2654748.874944] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2654748.952256] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2654749.025208] CPU: 1 PID: 23880 Comm: ll_ost01_040 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2654749.038615] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2654749.047030] task: ffff9c2c947ac1c0 ti: ffff9c152ca4c000 task.ti: ffff9c152ca4c000 [2654749.055364] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2654749.066160] RSP: 0018:ffff9c152ca4fb68 EFLAGS: 00000246 [2654749.072345] RAX: 0000000000000000 RBX: ffff9c152ca4fb18 RCX: 0000000000090000 [2654749.080348] RDX: ffff9c2d9d8db880 RSI: 0000000000b90101 RDI: ffff9c2a4a5112dc [2654749.088347] RBP: ffff9c152ca4fb68 R08: ffff9c2d9d61b880 R09: 0000000000000000 [2654749.096352] R10: ffff9c147036d800 R11: ffff9c146f562280 R12: ffff9c24ce83a850 [2654749.104367] R13: ffff9c29d22b4400 R14: 00000000c29db919 R15: 0000000000000000 [2654749.112382] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2654749.121356] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2654749.127994] CR2: 000000c000ef3010 CR3: 0000001726826000 CR4: 00000000007607e0 [2654749.136028] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2654749.144058] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2654749.152081] PKRU: 00000000 [2654749.155688] Call Trace: [2654749.159037] [] queued_spin_lock_slowpath+0xb/0xf [2654749.166192] [] _raw_spin_lock+0x20/0x30 [2654749.172582] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2654749.180016] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2654749.187631] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2654749.195235] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2654749.203184] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2654749.211575] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2654749.218584] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2654749.226371] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2654749.234833] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2654749.242790] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2654749.251355] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2654749.259040] [] ? __wake_up+0x13/0x20 [2654749.265180] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2654749.272349] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2654749.280609] [] kthread+0xd1/0xe0 [2654749.286352] [] ? insert_kthread_work+0x40/0x40 [2654749.293294] [] ret_from_fork_nospec_begin+0x7/0x21 [2654749.300564] [] ? insert_kthread_work+0x40/0x40 [2654749.307478] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2654749.330316] Lustre: f2-OST0038: Client f2-MDT0003-mdtlov_UUID (at 10.10.33.9@o2ib2) reconnecting [2654749.330382] Lustre: f2-OST0014: Connection restored to f2-MDT0002-mdtlov_UUID (at 10.10.33.8@o2ib2) [2654749.330383] Lustre: Skipped 46 previous similar messages [2654749.356040] Lustre: Skipped 8 previous similar messages [2654770.252717] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000402:3499434 to 0x7c0000402:3499649 [2654770.269827] Lustre: f2-OST0038: deleting orphan objects from 0x940000402:3497674 to 0x940000402:3497889 [2654771.141861] Lustre: f2-OST0038: deleting orphan objects from 0x940000401:4568712 to 0x940000401:4568801 [2654771.158888] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000401:4566888 to 0x7c0000401:4566913 [2654774.765859] Lustre: f2-OST0038: deleting orphan objects from 0x0:6901136 to 0x0:6901322 [2654777.429745] Lustre: f2-OST0038: deleting orphan objects from 0x940000400:5471224 to 0x940000400:5471313 [2654777.446830] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000400:5469723 to 0x7c0000400:5469812 [2654799.792877] Lustre: f2-OST0014: Client f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) reconnecting [2654799.802448] Lustre: Skipped 7 previous similar messages [2654799.808547] Lustre: f2-OST0014: deleting orphan objects from 0x0:6902435 to 0x0:6902657 [2656107.018668] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_ost01_107:24771] [2656107.075529] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ldlm_cn01_007:30413] [2656107.027362] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables [2656107.075530] Modules linked in: [2656107.075531] lustre(OE) [2656107.075531] lmv(OE) [2656107.075532] mdc(OE) [2656107.075532] osc(OE) [2656107.075532] lov(OE) [2656107.075533] osp(OE) [2656107.075533] ofd(OE) [2656107.075533] lfsck(OE) [2656107.075534] ost(OE) [2656107.075534] mgc(OE) [2656107.075534] osd_zfs(OE) [2656107.075535] lquota(OE) [2656107.075535] fid(OE) [2656107.075535] fld(OE) [2656107.075536] ptlrpc(OE) [2656107.075536] obdclass(OE) [2656107.075536] ko2iblnd(OE) [2656107.075537] dm_round_robin [2656107.075537] dm_multipath [2656107.075538] lnet(OE) [2656107.075538] libcfs(OE) [2656107.075538] xt_comment [2656107.075539] xt_multiport [2656107.075539] zfs(POE) [2656107.075539] zunicode(POE) [2656107.075540] zlua(POE) [2656107.075540] zcommon(POE) [2656107.075541] znvpair(POE) [2656107.075541] zavl(POE) [2656107.075541] icp(POE) [2656107.075542] spl(OE) [2656107.075542] sd_mod [2656107.075542] sg [2656107.075543] nfsv3 [2656107.075543] nfs_acl [2656107.075543] mpt2sas [2656107.075544] raid_class [2656107.075544] scsi_transport_sas [2656107.075545] mptctl [2656107.075545] mptbase [2656107.075545] ip6t_REJECT [2656107.075546] nf_reject_ipv6 [2656107.075546] nf_conntrack_ipv6 [2656107.075546] nf_defrag_ipv6 [2656107.075547] ip6table_filter [2656107.075547] ip6_tables [2656107.075548] ipt_REJECT [2656107.075548] nf_reject_ipv4 [2656107.075548] nf_conntrack_ipv4 [2656107.075549] nf_defrag_ipv4 [2656107.075549] xt_conntrack [2656107.075550] nf_conntrack [2656107.075550] libcrc32c [2656107.075551] iptable_filter [2656107.075552] dell_rbu [2656107.075553] ib_isert [2656107.075553] iscsi_target_mod [2656107.075554] target_core_mod [2656107.075554] crc_t10dif [2656107.075555] crct10dif_generic [2656107.075555] ib_srp [2656107.075556] scsi_transport_srp [2656107.075556] scsi_tgt [2656107.075557] rpcrdma [2656107.075557] rdma_ucm [2656107.075558] ib_iser [2656107.075558] ib_umad [2656107.075558] rdma_cm [2656107.075559] ib_ipoib [2656107.075559] iw_cm [2656107.075560] libiscsi [2656107.075560] scsi_transport_iscsi [2656107.075561] ib_cm [2656107.075561] mlx5_ib [2656107.075562] ib_uverbs [2656107.075562] ib_core [2656107.075562] dm_mirror [2656107.075563] dm_region_hash [2656107.075563] dm_log [2656107.075564] dm_mod [2656107.075564] skx_edac [2656107.075564] intel_powerclamp [2656107.075565] coretemp [2656107.075565] intel_rapl [2656107.075566] iosf_mbi [2656107.075566] kvm_intel [2656107.075567] kvm [2656107.075567] irqbypass [2656107.075568] iTCO_wdt [2656107.075568] crct10dif_pclmul [2656107.075569] iTCO_vendor_support [2656107.075569] crct10dif_common [2656107.075570] crc32_pclmul [2656107.075570] dell_smbios [2656107.075570] ghash_clmulni_intel [2656107.075571] dell_wmi_descriptor [2656107.075571] dcdbas [2656107.075572] aesni_intel [2656107.075572] lrw [2656107.075573] gf128mul [2656107.075573] glue_helper [2656107.075574] ablk_helper [2656107.075574] cryptd [2656107.075575] pcspkr [2656107.075575] mgag200 [2656107.075576] ttm [2656107.075576] drm_kms_helper [2656107.075577] syscopyarea [2656107.075577] sysfillrect [2656107.075578] sysimgblt [2656107.075578] fb_sys_fops [2656107.075579] drm [2656107.075579] drm_panel_orientation_quirks [2656107.075580] lpc_ich [2656107.075580] i2c_i801 [2656107.075581] ahci [2656107.075581] libahci [2656107.075582] libata [2656107.075582] mei_me [2656107.075583] mei [2656107.075583] wmi [2656107.075584] ipmi_si [2656107.075584] ipmi_devintf [2656107.075585] ipmi_msghandler [2656107.075585] nfit [2656107.075586] libnvdimm [2656107.075586] acpi_power_meter [2656107.075587] acpi_pad [2656107.075587] pcc_cpufreq [2656107.075588] binfmt_misc [2656107.075588] ip_tables [2656107.075589] rpcsec_gss_krb5 [2656107.075589] auth_rpcgss [2656107.075590] nfsv4 [2656107.075590] dns_resolver [2656107.075591] nfs [2656107.075591] lockd [2656107.075592] grace [2656107.075592] fscache [2656107.075593] mlx5_core [2656107.075593] igb [2656107.075594] crc32c_intel [2656107.075594] mlxfw [2656107.075595] i2c_algo_bit [2656107.075595] devlink [2656107.075595] dca [2656107.075596] ptp [2656107.075596] pps_core [2656107.075597] sunrpc [2656107.075597] [2656107.075599] CPU: 17 PID: 30413 Comm: ldlm_cn01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2656107.075600] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2656107.075600] task: ffff9c154e7b9070 ti: ffff9c2be25c4000 task.ti: ffff9c2be25c4000 [2656107.075601] RIP: 0010:[] [2656107.075638] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2656107.075639] RSP: 0018:ffff9c2be25c7ac0 EFLAGS: 00000283 [2656107.075639] RAX: ffff9c1e1d2b8480 RBX: ffff9c2be25c7a70 RCX: 000000000000001b [2656107.075640] RDX: 0000000000000004 RSI: 000000000480e000 RDI: ffff9c144e5f4800 [2656107.075640] RBP: ffff9c2be25c7b48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2656107.075641] R10: 000000000480e000 R11: 000000000480e000 R12: ffff9c24128ab200 [2656107.075641] R13: ffff9c2be25c7ab0 R14: ffffffffffffff10 R15: ffff9c24128ab200 [2656107.075642] FS: 0000000000000000(0000) GS:ffff9c2d9d800000(0000) knlGS:0000000000000000 [2656107.075643] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2656107.075644] CR2: 000000c00238f000 CR3: 000000190da10000 CR4: 00000000007607e0 [2656107.075645] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2656107.075645] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2656107.075646] PKRU: 00000000 [2656107.075646] Call Trace: [2656107.075665] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2656107.075681] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2656107.075696] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2656107.075707] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2656107.075722] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2656107.075736] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2656107.075754] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2656107.075771] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2656107.075787] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2656107.075810] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2656107.075830] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2656107.075833] [] ? __wake_up+0x13/0x20 [2656107.075854] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2656107.075874] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2656107.075876] [] kthread+0xd1/0xe0 [2656107.075878] [] ? insert_kthread_work+0x40/0x40 [2656107.075880] [] ret_from_fork_nospec_begin+0x7/0x21 [2656107.075882] [] ? insert_kthread_work+0x40/0x40 [2656107.075882] Code: [2656107.075883] b8 [2656107.075883] 00 [2656107.075884] 00 [2656107.075884] 00 [2656107.075884] 74 [2656107.075884] 2f [2656107.075885] 48 [2656107.075885] 8b [2656107.075885] 50 [2656107.075885] 60 [2656107.075886] 49 [2656107.075886] 39 [2656107.075886] d0 [2656107.075886] 48 [2656107.075887] 8d [2656107.075887] 42 [2656107.075887] a0 [2656107.075887] 0f [2656107.075888] 84 [2656107.075888] c7 [2656107.075888] 00 [2656107.075889] 00 [2656107.075889] 00 [2656107.075889] 48 [2656107.075889] 39 [2656107.075890] f3 [2656107.075890] 0f [2656107.075890] 84 [2656107.075890] de [2656107.075891] 02 [2656107.075891] 00 [2656107.075891] 00 [2656107.075891] 49 [2656107.075892] 39 [2656107.075892] c7 [2656107.075892] 74 [2656107.075892] e1 [2656107.075893] 8b [2656107.075893] 90 [2656107.075893] 98 [2656107.075894] 00 [2656107.075894] 00 [2656107.075894] 00 [2656107.075894] <44> [2656107.075895] 85 [2656107.075895] 24 [2656107.075895] 95 [2656107.075895] 00 [2656107.075896] 01 [2656107.075896] 5a [2656107.075896] c1 [2656107.075896] 75 [2656107.075897] c1 [2656107.075897] 83 [2656107.075897] c1 [2656107.075898] 01 [2656107.075898] 48 [2656107.075898] 89 [2656107.075898] f7 [2656107.075899] 48 [2656107.075899] 8b [2656107.075899] 90 [2656107.075899] d0 [2656107.075900] 00 [2656107.075900] [2656107.529521] ipt_REJECT [2656107.532815] nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me [2656107.607574] mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2656107.633756] CPU: 3 PID: 24771 Comm: ll_ost01_107 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2656107.647130] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2656107.655524] task: ffff9c143ae01070 ti: ffff9c143b478000 task.ti: ffff9c143b478000 [2656107.663831] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1cc/0x200 [2656107.674596] RSP: 0018:ffff9c143b47bad0 EFLAGS: 00000202 [2656107.680736] RAX: 0000000000000001 RBX: ffff9c26bf594850 RCX: 0000000000000001 [2656107.688697] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2656107.696658] RBP: ffff9c143b47bad0 R08: 0000000000000101 R09: 0000000000000158 [2656107.704621] R10: ffff9bff3fc07500 R11: ffff9c14498ade40 R12: ffffffffc14b9237 [2656107.712585] R13: ffff9c143b47ba58 R14: 0000000000000003 R15: 0000000000000003 [2656107.720548] FS: 0000000000000000(0000) GS:ffff9c2d9d640000(0000) knlGS:0000000000000000 [2656107.729470] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2656107.736048] CR2: 000000c00241d000 CR3: 000000190da10000 CR4: 00000000007607e0 [2656107.744020] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2656107.751993] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2656107.759962] PKRU: 00000000 [2656107.763516] Call Trace: [2656107.766805] [] queued_spin_lock_slowpath+0xb/0xf [2656107.773910] [] _raw_spin_lock+0x20/0x30 [2656107.780239] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2656107.787542] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2656107.795085] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2656107.803058] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2656107.810273] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2656107.818180] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2656107.826529] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2656107.833485] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2656107.841214] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2656107.849622] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2656107.857536] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2656107.866058] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2656107.873691] [] ? __wake_up+0x13/0x20 [2656107.879790] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2656107.886919] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2656107.895167] [] kthread+0xd1/0xe0 [2656107.900891] [] ? insert_kthread_work+0x40/0x40 [2656107.907814] [] ret_from_fork_nospec_begin+0x7/0x21 [2656107.915075] [] ? insert_kthread_work+0x40/0x40 [2656107.921975] Code: 90 8b 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c 90 8b 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d [2660368.719220] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_ost00_032:24595] [2660368.723210] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ll_ost01_088:24731] [2660368.723244] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660368.723277] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660368.723279] CPU: 21 PID: 24731 Comm: ll_ost01_088 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660368.723280] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660368.723281] task: ffff9c1598881070 ti: ffff9c143192c000 task.ti: ffff9c143192c000 [2660368.723287] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660368.723287] RSP: 0018:ffff9c143192fb68 EFLAGS: 00000246 [2660368.723288] RAX: 0000000000000000 RBX: ffff9c143192fb18 RCX: 0000000000a90000 [2660368.723289] RDX: ffff9c2d9d81b880 RSI: 0000000000890101 RDI: ffff9c2a4a5112dc [2660368.723289] RBP: ffff9c143192fb68 R08: ffff9c2d9d89b880 R09: 0000000000000000 [2660368.723290] R10: ffff9c147036d800 R11: ffff9c2c589bce40 R12: ffff9c2095999850 [2660368.723290] R13: ffff9c1fe1c69800 R14: 00000000d8f42961 R15: 0000000000000000 [2660368.723291] FS: 0000000000000000(0000) GS:ffff9c2d9d880000(0000) knlGS:0000000000000000 [2660368.723292] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660368.723292] CR2: 000000c0020cd010 CR3: 000000172631c000 CR4: 00000000007607e0 [2660368.723293] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660368.723294] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660368.723294] PKRU: 00000000 [2660368.723295] Call Trace: [2660368.723300] [] queued_spin_lock_slowpath+0xb/0xf [2660368.723303] [] _raw_spin_lock+0x20/0x30 [2660368.723337] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660368.723352] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660368.723375] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660368.723393] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660368.723413] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660368.723441] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660368.723466] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660368.723488] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660368.723496] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660368.723516] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660368.723536] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660368.723539] [] ? __wake_up+0x13/0x20 [2660368.723559] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660368.723578] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660368.723580] [] kthread+0xd1/0xe0 [2660368.723582] [] ? insert_kthread_work+0x40/0x40 [2660368.723584] [] ret_from_fork_nospec_begin+0x7/0x21 [2660368.723585] [] ? insert_kthread_work+0x40/0x40 [2660368.723599] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660368.727200] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ll_ost00_058:24716] [2660368.727234] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660368.727264] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660368.727266] CPU: 22 PID: 24716 Comm: ll_ost00_058 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660368.727267] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660368.727268] task: ffff9c152b238000 ti: ffff9c142fef4000 task.ti: ffff9c142fef4000 [2660368.727274] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660368.727275] RSP: 0018:ffff9c142fef7b68 EFLAGS: 00000246 [2660368.727275] RAX: 0000000000000000 RBX: ffff9c142fef7b18 RCX: 0000000000b10000 [2660368.727276] RDX: ffff9c159f3db880 RSI: 0000000000710101 RDI: ffff9c2a4a5112dc [2660368.727277] RBP: ffff9c142fef7b68 R08: ffff9c159f4db880 R09: 0000000000000000 [2660368.727277] R10: ffff9c147036d800 R11: ffff9c144c6c7080 R12: ffff9c146b4a6850 [2660368.727278] R13: ffff9c1466fbc800 R14: 0000000044f3c1d8 R15: 0000000000000000 [2660368.727279] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2660368.727279] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660368.727280] CR2: 00007fa34cc20000 CR3: 000000172e636000 CR4: 00000000007607e0 [2660368.727281] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660368.727281] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660368.727282] PKRU: 00000000 [2660368.727282] Call Trace: [2660368.727288] [] queued_spin_lock_slowpath+0xb/0xf [2660368.727292] [] _raw_spin_lock+0x20/0x30 [2660368.727335] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660368.727350] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660368.727374] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660368.727391] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660368.727412] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660368.727441] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660368.727466] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660368.727489] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660368.727495] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660368.727515] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660368.727535] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660368.727538] [] ? __wake_up+0x13/0x20 [2660368.727558] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660368.727577] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660368.727580] [] kthread+0xd1/0xe0 [2660368.727581] [] ? insert_kthread_work+0x40/0x40 [2660368.727584] [] ret_from_fork_nospec_begin+0x7/0x21 [2660368.727585] [] ? insert_kthread_work+0x40/0x40 [2660368.727599] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660368.731191] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_ost01_037:23838] [2660368.731217] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660368.731241] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660368.731243] CPU: 23 PID: 23838 Comm: ll_ost01_037 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660368.731243] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660368.731244] task: ffff9bff19d51070 ti: ffff9c1455594000 task.ti: ffff9c1455594000 [2660368.731248] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660368.731249] RSP: 0018:ffff9c1455597b68 EFLAGS: 00000246 [2660368.731249] RAX: 0000000000000000 RBX: ffff9c1455597b18 RCX: 0000000000b90000 [2660368.731250] RDX: ffff9c2d9d89b880 RSI: 0000000000a90101 RDI: ffff9c2a4a5112dc [2660368.731250] RBP: ffff9c1455597b68 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2660368.731251] R10: ffff9c147036d800 R11: ffff9c2d986f9640 R12: ffff9c286d192850 [2660368.731251] R13: ffff9c28a23d8c00 R14: 00000000da498892 R15: 0000000000000000 [2660368.731252] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2660368.731253] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660368.731254] CR2: 000000c00220f010 CR3: 000000172631c000 CR4: 00000000007607e0 [2660368.731254] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660368.731255] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660368.731255] PKRU: 00000000 [2660368.731256] Call Trace: [2660368.731260] [] queued_spin_lock_slowpath+0xb/0xf [2660368.731263] [] _raw_spin_lock+0x20/0x30 [2660368.731291] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660368.731306] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660368.731327] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660368.731345] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660368.731365] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660368.731391] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660368.731415] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660368.731437] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660368.731442] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660368.731462] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660368.731482] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660368.731484] [] ? wake_up_state+0x20/0x20 [2660368.731503] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660368.731523] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660368.731525] [] kthread+0xd1/0xe0 [2660368.731526] [] ? insert_kthread_work+0x40/0x40 [2660368.731528] [] ret_from_fork_nospec_begin+0x7/0x21 [2660368.731529] [] ? insert_kthread_work+0x40/0x40 [2660368.731543] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660370.100313] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660370.177615] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660370.250561] CPU: 20 PID: 24595 Comm: ll_ost00_032 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660370.264045] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660370.272458] task: ffff9c1597c062a0 ti: ffff9c1439ee4000 task.ti: ffff9c1439ee4000 [2660370.280792] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660370.291591] RSP: 0018:ffff9c1439ee7b68 EFLAGS: 00000246 [2660370.297773] RAX: 0000000000000000 RBX: ffff9c1439ee7b18 RCX: 0000000000a10000 [2660370.305779] RDX: ffff9c159f21b880 RSI: 0000000000010101 RDI: ffff9c2a4a5112dc [2660370.313777] RBP: ffff9c1439ee7b68 R08: ffff9c159f49b880 R09: 0000000000000000 [2660370.321782] R10: ffff9c147036d800 R11: ffff9c144c6dcac0 R12: ffff9c0f97f4c050 [2660370.329788] R13: ffff9c1460af1400 R14: 00000000290bc8f1 R15: 0000000000000000 [2660370.337793] FS: 0000000000000000(0000) GS:ffff9c159f480000(0000) knlGS:0000000000000000 [2660370.346768] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660370.353400] CR2: 00007f8e4e3c1bc0 CR3: 000000172e636000 CR4: 00000000007607e0 [2660370.361423] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660370.369446] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660370.377470] PKRU: 00000000 [2660370.381072] Call Trace: [2660370.384415] [] queued_spin_lock_slowpath+0xb/0xf [2660370.391572] [] _raw_spin_lock+0x20/0x30 [2660370.397964] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660370.405393] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660370.412999] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660370.420594] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660370.428543] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660370.436934] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660370.443935] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660370.451714] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660370.460169] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660370.468121] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660370.476689] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660370.484365] [] ? __wake_up+0x13/0x20 [2660370.490507] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660370.497674] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660370.505936] [] kthread+0xd1/0xe0 [2660370.511677] [] ? insert_kthread_work+0x40/0x40 [2660370.518619] [] ret_from_fork_nospec_begin+0x7/0x21 [2660370.525891] [] ? insert_kthread_work+0x40/0x40 [2660370.532807] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.631701] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ll_ost01_093:24744] [2660372.632701] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ll_ost00_009:22888] [2660372.632722] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.632743] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.632744] CPU: 0 PID: 22888 Comm: ll_ost00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.632745] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.632746] task: ffff9c148bcd9070 ti: ffff9c2c8640c000 task.ti: ffff9c2c8640c000 [2660372.632749] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.632749] RSP: 0018:ffff9c2c8640fb68 EFLAGS: 00000246 [2660372.632750] RAX: 0000000000000000 RBX: ffff9c2c8640fb18 RCX: 0000000000010000 [2660372.632751] RDX: ffff9c2d9d8db880 RSI: 0000000000b90101 RDI: ffff9c2a4a5112dc [2660372.632751] RBP: ffff9c2c8640fb68 R08: ffff9c159f21b880 R09: 0000000000000000 [2660372.632752] R10: ffff9c147036d800 R11: ffff9c14878b5040 R12: ffff9c0324901850 [2660372.632752] R13: ffff9c295a076400 R14: 000000008ddae1ab R15: 0000000000000000 [2660372.632753] FS: 0000000000000000(0000) GS:ffff9c159f200000(0000) knlGS:0000000000000000 [2660372.632754] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.632755] CR2: 000000c002329000 CR3: 000000172e636000 CR4: 00000000007607f0 [2660372.632755] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.632756] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.632756] PKRU: 00000000 [2660372.632756] Call Trace: [2660372.632759] [] queued_spin_lock_slowpath+0xb/0xf [2660372.632761] [] _raw_spin_lock+0x20/0x30 [2660372.632779] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.632794] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.632814] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.632832] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.632852] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.632877] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.632901] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.632924] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.632928] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.632948] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.632968] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.632970] [] ? __wake_up+0x13/0x20 [2660372.632989] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.633009] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.633010] [] kthread+0xd1/0xe0 [2660372.633012] [] ? insert_kthread_work+0x40/0x40 [2660372.633014] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.633015] [] ? insert_kthread_work+0x40/0x40 [2660372.633029] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.636689] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ll_ost00_067:24733] [2660372.636711] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.636732] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.636733] CPU: 2 PID: 24733 Comm: ll_ost00_067 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.636734] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.636735] task: ffff9c14e4709070 ti: ffff9c1431ea4000 task.ti: ffff9c1431ea4000 [2660372.636738] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2660372.636738] RSP: 0018:ffff9c1431ea7b68 EFLAGS: 00000246 [2660372.636739] RAX: 0000000000000000 RBX: ffff9c1431ea7b18 RCX: 0000000000110000 [2660372.636739] RDX: ffff9c159f39b880 RSI: 0000000000610101 RDI: ffff9c2a4a5112dc [2660372.636740] RBP: ffff9c1431ea7b68 R08: ffff9c159f25b880 R09: 0000000000000000 [2660372.636741] R10: ffff9c147036d800 R11: ffff9c2d99d1a900 R12: ffff9c145a897850 [2660372.636741] R13: ffff9bfdde644800 R14: 00000000b6891863 R15: 0000000000000000 [2660372.636742] FS: 0000000000000000(0000) GS:ffff9c159f240000(0000) knlGS:0000000000000000 [2660372.636743] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.636743] CR2: 000000c002203010 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.636744] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.636745] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.636745] PKRU: 00000000 [2660372.636745] Call Trace: [2660372.636748] [] queued_spin_lock_slowpath+0xb/0xf [2660372.636750] [] _raw_spin_lock+0x20/0x30 [2660372.636768] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.636782] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.636803] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.636821] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.636840] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.636866] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.636890] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.636912] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.636916] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.636936] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.636956] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.636958] [] ? __wake_up+0x13/0x20 [2660372.636977] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.636997] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.636998] [] kthread+0xd1/0xe0 [2660372.637000] [] ? insert_kthread_work+0x40/0x40 [2660372.637002] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.637003] [] ? insert_kthread_work+0x40/0x40 [2660372.637017] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2660372.640680] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [ll_ost01_003:22866] [2660372.640705] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.640728] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.640730] CPU: 3 PID: 22866 Comm: ll_ost01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.640730] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.640731] task: ffff9c2c8bbb0000 ti: ffff9c2c94d30000 task.ti: ffff9c2c94d30000 [2660372.640734] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2660372.640735] RSP: 0018:ffff9c2c94d33b68 EFLAGS: 00000202 [2660372.640736] RAX: 0000000000000001 RBX: ffff9c2c94d33b18 RCX: 0000000000000001 [2660372.640736] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2660372.640737] RBP: ffff9c2c94d33b68 R08: 0000000000000101 R09: 0000000000000000 [2660372.640737] R10: ffff9c147036d800 R11: ffff9c146ed5dd80 R12: ffff9c1c76150850 [2660372.640738] R13: ffff9c2a9f00fc00 R14: 00000000b7aed388 R15: 0000000000000000 [2660372.640739] FS: 0000000000000000(0000) GS:ffff9c2d9d640000(0000) knlGS:0000000000000000 [2660372.640739] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.640740] CR2: 000000c002265000 CR3: 0000002fa937e000 CR4: 00000000007607e0 [2660372.640741] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.640741] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.640742] PKRU: 00000000 [2660372.640742] Call Trace: [2660372.640746] [] queued_spin_lock_slowpath+0xb/0xf [2660372.640748] [] _raw_spin_lock+0x20/0x30 [2660372.640773] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.640788] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.640810] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.640828] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.640850] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.640877] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.640903] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.640926] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.640931] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.640952] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.640973] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.640975] [] ? wake_up_state+0x20/0x20 [2660372.640996] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.641016] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.641018] [] kthread+0xd1/0xe0 [2660372.641019] [] ? insert_kthread_work+0x40/0x40 [2660372.641021] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.641022] [] ? insert_kthread_work+0x40/0x40 [2660372.641037] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2660372.643672] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [ll_ost00_104:24809] [2660372.643694] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.643714] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.643717] CPU: 4 PID: 24809 Comm: ll_ost00_104 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.643717] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.643718] task: ffff9c144d3541c0 ti: ffff9c144cd78000 task.ti: ffff9c144cd78000 [2660372.643721] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2660372.643721] RSP: 0018:ffff9c144cd7bb68 EFLAGS: 00000246 [2660372.643722] RAX: 0000000000000000 RBX: ffff9c144cd7bb18 RCX: 0000000000210000 [2660372.643723] RDX: ffff9c159f4db880 RSI: 0000000000b10101 RDI: ffff9c2a4a5112dc [2660372.643723] RBP: ffff9c144cd7bb68 R08: ffff9c159f29b880 R09: 0000000000000000 [2660372.643724] R10: ffff9c147036d800 R11: ffff9c144a9e8c80 R12: ffff9c146aa42050 [2660372.643724] R13: ffff9c0847979400 R14: 000000004e2de354 R15: 0000000000000000 [2660372.643725] FS: 0000000000000000(0000) GS:ffff9c159f280000(0000) knlGS:0000000000000000 [2660372.643726] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.643726] CR2: 00007f9d0f438000 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.643727] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.643728] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.643728] PKRU: 00000000 [2660372.643728] Call Trace: [2660372.643731] [] queued_spin_lock_slowpath+0xb/0xf [2660372.643733] [] _raw_spin_lock+0x20/0x30 [2660372.643748] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.643763] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.643784] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.643801] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.643822] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.643847] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.643871] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.643893] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.643897] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.643917] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.643937] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.643939] [] ? __wake_up+0x13/0x20 [2660372.643959] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.643978] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.643980] [] kthread+0xd1/0xe0 [2660372.643982] [] ? insert_kthread_work+0x40/0x40 [2660372.643983] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.643984] [] ? insert_kthread_work+0x40/0x40 [2660372.643999] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2660372.647663] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [ll_ost01_007:22882] [2660372.647686] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.647708] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.647710] CPU: 5 PID: 22882 Comm: ll_ost01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.647710] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.647711] task: ffff9c2d99c062a0 ti: ffff9c2c8b4a8000 task.ti: ffff9c2c8b4a8000 [2660372.647714] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.647715] RSP: 0018:ffff9c2c8b4abb68 EFLAGS: 00000246 [2660372.647715] RAX: 0000000000000000 RBX: ffff9c2c8b4abb18 RCX: 0000000000290000 [2660372.647716] RDX: ffff9c159f49b880 RSI: 0000000000a10101 RDI: ffff9c2a4a5112dc [2660372.647716] RBP: ffff9c2c8b4abb68 R08: ffff9c2d9d69b880 R09: 0000000000000000 [2660372.647717] R10: ffff9c147036d800 R11: ffff9c157e365340 R12: ffff9c2095998050 [2660372.647717] R13: ffff9c2d7226b000 R14: 0000000022d54d9e R15: 0000000000000000 [2660372.647718] FS: 0000000000000000(0000) GS:ffff9c2d9d680000(0000) knlGS:0000000000000000 [2660372.647719] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.647720] CR2: 00007fcb82c08000 CR3: 000000190da10000 CR4: 00000000007607e0 [2660372.647720] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.647721] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.647721] PKRU: 00000000 [2660372.647722] Call Trace: [2660372.647725] [] queued_spin_lock_slowpath+0xb/0xf [2660372.647727] [] _raw_spin_lock+0x20/0x30 [2660372.647749] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.647764] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.647785] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.647803] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.647823] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.647848] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.647873] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.647896] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.647900] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.647920] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.647940] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.647943] [] ? __wake_up+0x13/0x20 [2660372.647962] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.647982] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.647983] [] kthread+0xd1/0xe0 [2660372.647985] [] ? insert_kthread_work+0x40/0x40 [2660372.647987] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.647988] [] ? insert_kthread_work+0x40/0x40 [2660372.648002] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.652650] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ll_ost00_038:24607] [2660372.652677] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.652700] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.652702] CPU: 6 PID: 24607 Comm: ll_ost00_038 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.652703] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.652704] task: ffff9c1551e720e0 ti: ffff9c143e1ac000 task.ti: ffff9c143e1ac000 [2660372.652707] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.652708] RSP: 0018:ffff9c143e1afb68 EFLAGS: 00000246 [2660372.652708] RAX: 0000000000000000 RBX: ffff9c143e1afb18 RCX: 0000000000310000 [2660372.652709] RDX: ffff9c159f35b880 RSI: 0000000000510101 RDI: ffff9c2a4a5112dc [2660372.652710] RBP: ffff9c143e1afb68 R08: ffff9c159f2db880 R09: 0000000000000000 [2660372.652711] R10: ffff9c147036d800 R11: ffff9c144c6dcfc0 R12: ffff9c145173b050 [2660372.652711] R13: ffff9c133da51c00 R14: 00000000341b14ee R15: 0000000000000000 [2660372.652712] FS: 0000000000000000(0000) GS:ffff9c159f2c0000(0000) knlGS:0000000000000000 [2660372.652713] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.652713] CR2: 00007f556dcf9000 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.652714] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.652715] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.652716] PKRU: 00000000 [2660372.652716] Call Trace: [2660372.652719] [] queued_spin_lock_slowpath+0xb/0xf [2660372.652721] [] _raw_spin_lock+0x20/0x30 [2660372.652737] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.652752] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.652773] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.652791] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.652811] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.652836] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.652861] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.652883] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.652888] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.652908] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.652928] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.652930] [] ? __wake_up+0x13/0x20 [2660372.652950] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.652970] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.652972] [] kthread+0xd1/0xe0 [2660372.652974] [] ? insert_kthread_work+0x40/0x40 [2660372.652975] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.652977] [] ? insert_kthread_work+0x40/0x40 [2660372.652996] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.656641] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ll_ost01_137:25265] [2660372.656664] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.656686] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.656688] CPU: 7 PID: 25265 Comm: ll_ost01_137 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.656688] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.656689] task: ffff9c2c4fb33150 ti: ffff9c2c64294000 task.ti: ffff9c2c64294000 [2660372.656693] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.656694] RSP: 0018:ffff9c2c64297b68 EFLAGS: 00000246 [2660372.656694] RAX: 0000000000000000 RBX: ffff9c2c64297b18 RCX: 0000000000390000 [2660372.656695] RDX: ffff9c2d9d71b880 RSI: 0000000000490101 RDI: ffff9c2a4a5112dc [2660372.656695] RBP: ffff9c2c64297b68 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2660372.656696] R10: ffff9c147036d800 R11: ffff9c1486cfcb00 R12: ffff9c209599c850 [2660372.656696] R13: ffff9c25f682ec00 R14: 0000000063ea0513 R15: 0000000000000000 [2660372.656697] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2660372.656698] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.656699] CR2: 00007f76f4e82000 CR3: 000000190da10000 CR4: 00000000007607e0 [2660372.656699] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.656700] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.656700] PKRU: 00000000 [2660372.656700] Call Trace: [2660372.656704] [] queued_spin_lock_slowpath+0xb/0xf [2660372.656706] [] _raw_spin_lock+0x20/0x30 [2660372.656729] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.656744] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.656766] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.656784] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.656804] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.656830] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.656854] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.656877] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.656881] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.656902] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.656922] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.656924] [] ? __wake_up+0x13/0x20 [2660372.656944] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.656964] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.656965] [] kthread+0xd1/0xe0 [2660372.656967] [] ? insert_kthread_work+0x40/0x40 [2660372.656969] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.656970] [] ? insert_kthread_work+0x40/0x40 [2660372.656984] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.660631] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [ll_ost00_092:24784] [2660372.660653] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.660674] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.660676] CPU: 8 PID: 24784 Comm: ll_ost00_092 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.660676] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.660677] task: ffff9c143d550000 ti: ffff9c1455194000 task.ti: ffff9c1455194000 [2660372.660680] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.660680] RSP: 0018:ffff9c1455197b68 EFLAGS: 00000246 [2660372.660681] RAX: 0000000000000000 RBX: ffff9c1455197b18 RCX: 0000000000410000 [2660372.660681] RDX: ffff9c2d9d7db880 RSI: 0000000000790101 RDI: ffff9c2a4a5112dc [2660372.660682] RBP: ffff9c1455197b68 R08: ffff9c159f31b880 R09: 0000000000000000 [2660372.660683] R10: ffff9c147036d800 R11: ffff9c144a9ec500 R12: ffff9c145b2bc850 [2660372.660683] R13: ffff9c145b325c00 R14: 00000000ac4579dc R15: 0000000000000000 [2660372.660684] FS: 0000000000000000(0000) GS:ffff9c159f300000(0000) knlGS:0000000000000000 [2660372.660685] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.660685] CR2: 000000c00164d000 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.660686] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.660686] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.660687] PKRU: 00000000 [2660372.660687] Call Trace: [2660372.660690] [] queued_spin_lock_slowpath+0xb/0xf [2660372.660692] [] _raw_spin_lock+0x20/0x30 [2660372.660707] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.660721] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.660742] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.660760] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.660780] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.660805] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.660829] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.660852] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.660855] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.660876] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.660895] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.660897] [] ? __wake_up+0x13/0x20 [2660372.660917] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.660936] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.660938] [] kthread+0xd1/0xe0 [2660372.660939] [] ? insert_kthread_work+0x40/0x40 [2660372.660941] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.660942] [] ? insert_kthread_work+0x40/0x40 [2660372.660957] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.664621] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ll_ost01_070:24699] [2660372.664644] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.664665] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.664667] CPU: 9 PID: 24699 Comm: ll_ost01_070 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.664667] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.664668] task: ffff9c153ef8c1c0 ti: ffff9c142e930000 task.ti: ffff9c142e930000 [2660372.664671] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.664672] RSP: 0018:ffff9c142e933b68 EFLAGS: 00000246 [2660372.664672] RAX: 0000000000000000 RBX: ffff9c142e933b18 RCX: 0000000000490000 [2660372.664673] RDX: ffff9c2d9d69b880 RSI: 0000000000290101 RDI: ffff9c2a4a5112dc [2660372.664673] RBP: ffff9c142e933b68 R08: ffff9c2d9d71b880 R09: 0000000000000000 [2660372.664674] R10: ffff9c147036d800 R11: ffff9c144c6b0e00 R12: ffff9c209599b850 [2660372.664674] R13: ffff9c2544a31800 R14: 000000003d81f273 R15: 0000000000000000 [2660372.664675] FS: 0000000000000000(0000) GS:ffff9c2d9d700000(0000) knlGS:0000000000000000 [2660372.664676] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.664677] CR2: 000000c00226b000 CR3: 000000172631c000 CR4: 00000000007607e0 [2660372.664677] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.664678] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.664678] PKRU: 00000000 [2660372.664678] Call Trace: [2660372.664681] [] queued_spin_lock_slowpath+0xb/0xf [2660372.664683] [] _raw_spin_lock+0x20/0x30 [2660372.664703] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.664718] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.664739] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.664757] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.664777] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.664803] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.664827] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.664850] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.664854] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.664875] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.664895] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.664897] [] ? __wake_up+0x13/0x20 [2660372.664916] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.664936] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.664938] [] kthread+0xd1/0xe0 [2660372.664939] [] ? insert_kthread_work+0x40/0x40 [2660372.664941] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.664942] [] ? insert_kthread_work+0x40/0x40 [2660372.664957] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.668611] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [ll_ost00_094:24788] [2660372.668633] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.668653] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.668655] CPU: 10 PID: 24788 Comm: ll_ost00_094 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.668655] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.668657] task: ffff9c143d5541c0 ti: ffff9c1441e84000 task.ti: ffff9c1441e84000 [2660372.668659] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.668660] RSP: 0018:ffff9c1441e87b68 EFLAGS: 00000246 [2660372.668661] RAX: 0000000000000000 RBX: ffff9c1441e87b18 RCX: 0000000000510000 [2660372.668661] RDX: ffff9c2d9d61b880 RSI: 0000000000090101 RDI: ffff9c2a4a5112dc [2660372.668662] RBP: ffff9c1441e87b68 R08: ffff9c159f35b880 R09: 0000000000000000 [2660372.668662] R10: ffff9c147036d800 R11: ffff9c144a9ec440 R12: ffff9c0f97f4b050 [2660372.668663] R13: ffff9c12cda47000 R14: 000000009b8005d3 R15: 0000000000000000 [2660372.668664] FS: 0000000000000000(0000) GS:ffff9c159f340000(0000) knlGS:0000000000000000 [2660372.668664] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.668665] CR2: 00000000009eb0a8 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.668666] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.668666] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.668667] PKRU: 00000000 [2660372.668667] Call Trace: [2660372.668670] [] queued_spin_lock_slowpath+0xb/0xf [2660372.668671] [] _raw_spin_lock+0x20/0x30 [2660372.668686] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.668701] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.668722] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.668739] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.668759] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.668785] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.668809] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.668831] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.668835] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.668856] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.668875] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.668877] [] ? wake_up_state+0x20/0x20 [2660372.668897] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.668917] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.668918] [] kthread+0xd1/0xe0 [2660372.668920] [] ? insert_kthread_work+0x40/0x40 [2660372.668921] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.668923] [] ? insert_kthread_work+0x40/0x40 [2660372.668937] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.672602] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ldlm_cn01_003:24627] [2660372.672625] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.672645] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.672647] CPU: 11 PID: 24627 Comm: ldlm_cn01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.672648] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.672649] task: ffff9c1521bcb150 ti: ffff9c145a4b4000 task.ti: ffff9c145a4b4000 [2660372.672671] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x224/0x810 [ptlrpc] [2660372.672672] RSP: 0018:ffff9c145a4b7ac0 EFLAGS: 00000206 [2660372.672672] RAX: ffff9c0d41e78480 RBX: ffff9c145a4b7a70 RCX: 0000000000000001 [2660372.672673] RDX: ffff9c118a7e8960 RSI: 0000000000000000 RDI: ffff9c146aae2000 [2660372.672673] RBP: ffff9c145a4b7b48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2660372.672674] R10: 00000000047e5000 R11: 00000000047e5000 R12: ffff9c1262020180 [2660372.672675] R13: ffff9c2a54bdb380 R14: ffffbf02b2874fa0 R15: ffff9c1262020180 [2660372.672675] FS: 0000000000000000(0000) GS:ffff9c2d9d740000(0000) knlGS:0000000000000000 [2660372.672676] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.672677] CR2: 000000c001255000 CR3: 000000172631c000 CR4: 00000000007607e0 [2660372.672677] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.672678] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.672678] PKRU: 00000000 [2660372.672679] Call Trace: [2660372.672697] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2660372.672712] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2660372.672727] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2660372.672733] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2660372.672748] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2660372.672762] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2660372.672779] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2660372.672796] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2660372.672812] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2660372.672833] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.672853] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.672855] [] ? __wake_up+0x13/0x20 [2660372.672875] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.672894] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.672896] [] kthread+0xd1/0xe0 [2660372.672898] [] ? insert_kthread_work+0x40/0x40 [2660372.672900] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.672901] [] ? insert_kthread_work+0x40/0x40 [2660372.672915] Code: 39 d0 48 8d 42 a0 0f 84 b8 04 00 00 48 8b 75 b8 31 c9 eb 24 0f 1f 00 49 8b bf b8 00 00 00 48 39 b8 b8 00 00 00 74 2f 48 8b 50 60 <49> 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 [2660372.676592] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ll_ost00_096:24791] [2660372.676615] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.676635] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.676637] CPU: 12 PID: 24791 Comm: ll_ost00_096 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.676637] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.676638] task: ffff9c1442810000 ti: ffff9c14428a8000 task.ti: ffff9c14428a8000 [2660372.676641] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.676642] RSP: 0018:ffff9c14428abb68 EFLAGS: 00000246 [2660372.676642] RAX: 0000000000000000 RBX: ffff9c14428abb18 RCX: 0000000000610000 [2660372.676643] RDX: ffff9c159f2db880 RSI: 0000000000310101 RDI: ffff9c2a4a5112dc [2660372.676644] RBP: ffff9c14428abb68 R08: ffff9c159f39b880 R09: 0000000000000000 [2660372.676644] R10: ffff9c147036d800 R11: ffff9c144a9ec2c0 R12: ffff9c146ac57050 [2660372.676645] R13: ffff9c02cf833000 R14: 00000000d4866086 R15: 0000000000000000 [2660372.676646] FS: 0000000000000000(0000) GS:ffff9c159f380000(0000) knlGS:0000000000000000 [2660372.676646] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.676647] CR2: 000000c000ecf000 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.676648] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.676648] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.676648] PKRU: 00000000 [2660372.676649] Call Trace: [2660372.676652] [] queued_spin_lock_slowpath+0xb/0xf [2660372.676653] [] _raw_spin_lock+0x20/0x30 [2660372.676671] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.676686] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.676707] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.676724] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.676744] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.676769] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.676794] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.676816] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.676820] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.676841] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.676861] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.676863] [] ? __wake_up+0x13/0x20 [2660372.676882] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.676901] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.676903] [] kthread+0xd1/0xe0 [2660372.676905] [] ? insert_kthread_work+0x40/0x40 [2660372.676906] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.676908] [] ? insert_kthread_work+0x40/0x40 [2660372.676921] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.680583] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ll_ost01_026:23803] [2660372.680606] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.680627] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.680629] CPU: 13 PID: 23803 Comm: ll_ost01_026 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.680630] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.680631] task: ffff9c2c944ba0e0 ti: ffff9c2cb0f1c000 task.ti: ffff9c2cb0f1c000 [2660372.680634] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2660372.680634] RSP: 0018:ffff9c2cb0f1fb68 EFLAGS: 00000202 [2660372.680635] RAX: 0000000000000101 RBX: ffff9c2cb0f1fb18 RCX: 0000000000690000 [2660372.680636] RDX: 0000000000810101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2660372.680636] RBP: ffff9c2cb0f1fb68 R08: ffff9c2d9d79b880 R09: 0000000000000000 [2660372.680637] R10: ffff9c147036d800 R11: ffff9c2d7e7347c0 R12: ffff9c256ed4a050 [2660372.680637] R13: ffff9c2c1a2a0800 R14: 00000000d126e9c8 R15: 0000000000000000 [2660372.680638] FS: 0000000000000000(0000) GS:ffff9c2d9d780000(0000) knlGS:0000000000000000 [2660372.680639] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.680639] CR2: 000055dfcfd82258 CR3: 00000017dbfc8000 CR4: 00000000007607e0 [2660372.680640] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.680641] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.680641] PKRU: 00000000 [2660372.680641] Call Trace: [2660372.680645] [] queued_spin_lock_slowpath+0xb/0xf [2660372.680646] [] _raw_spin_lock+0x20/0x30 [2660372.680667] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.680682] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.680702] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.680720] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.680741] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.680766] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.680790] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.680813] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.680817] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.680838] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.680858] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.680860] [] ? __wake_up+0x13/0x20 [2660372.680880] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.680900] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.680901] [] kthread+0xd1/0xe0 [2660372.680903] [] ? insert_kthread_work+0x40/0x40 [2660372.680904] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.680906] [] ? insert_kthread_work+0x40/0x40 [2660372.680920] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2660372.684572] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [ll_ost00_031:24594] [2660372.684595] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.684616] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.684618] CPU: 14 PID: 24594 Comm: ll_ost00_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.684618] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.684619] task: ffff9c1597ef1070 ti: ffff9c14542b0000 task.ti: ffff9c14542b0000 [2660372.684622] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.684622] RSP: 0018:ffff9c14542b3b68 EFLAGS: 00000246 [2660372.684623] RAX: 0000000000000000 RBX: ffff9c14542b3b18 RCX: 0000000000710000 [2660372.684624] RDX: ffff9c159f25b880 RSI: 0000000000110101 RDI: ffff9c2a4a5112dc [2660372.684624] RBP: ffff9c14542b3b68 R08: ffff9c159f3db880 R09: 0000000000000000 [2660372.684625] R10: ffff9c147036d800 R11: ffff9c144c6dcbc0 R12: ffff9c146aa41850 [2660372.684625] R13: ffff9c145b0e9000 R14: 00000000c5f0618a R15: 0000000000000000 [2660372.684626] FS: 0000000000000000(0000) GS:ffff9c159f3c0000(0000) knlGS:0000000000000000 [2660372.684627] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.684627] CR2: 000000c002225010 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.684628] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.684629] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.684629] PKRU: 00000000 [2660372.684629] Call Trace: [2660372.684632] [] queued_spin_lock_slowpath+0xb/0xf [2660372.684634] [] _raw_spin_lock+0x20/0x30 [2660372.684649] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.684664] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.684685] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.684702] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.684722] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.684747] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.684771] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.684793] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.684797] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.684817] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.684837] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.684839] [] ? __wake_up+0x13/0x20 [2660372.684858] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.684878] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.684880] [] kthread+0xd1/0xe0 [2660372.684881] [] ? insert_kthread_work+0x40/0x40 [2660372.684883] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.684884] [] ? insert_kthread_work+0x40/0x40 [2660372.684898] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.689560] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ll_ost01_015:23723] [2660372.689583] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.689603] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.689605] CPU: 15 PID: 23723 Comm: ll_ost01_015 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.689606] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.689606] task: ffff9c148e91c1c0 ti: ffff9c1537290000 task.ti: ffff9c1537290000 [2660372.689609] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2660372.689610] RSP: 0018:ffff9c1537293b68 EFLAGS: 00000246 [2660372.689611] RAX: 0000000000000000 RBX: ffff9c1537293b18 RCX: 0000000000790000 [2660372.689611] RDX: ffff9c2d9d79b880 RSI: 0000000000690101 RDI: ffff9c2a4a5112dc [2660372.689612] RBP: ffff9c1537293b68 R08: ffff9c2d9d7db880 R09: 0000000000000000 [2660372.689612] R10: ffff9c147036d800 R11: ffff9c2d7e734b80 R12: ffff9c1c76154050 [2660372.689613] R13: ffff9c1e93f21800 R14: 00000000e7b7efd4 R15: 0000000000000000 [2660372.689614] FS: 0000000000000000(0000) GS:ffff9c2d9d7c0000(0000) knlGS:0000000000000000 [2660372.689615] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.689615] CR2: 000000c000cb7000 CR3: 000000190da10000 CR4: 00000000007607e0 [2660372.689616] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.689616] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.689617] PKRU: 00000000 [2660372.689617] Call Trace: [2660372.689620] [] queued_spin_lock_slowpath+0xb/0xf [2660372.689622] [] _raw_spin_lock+0x20/0x30 [2660372.689640] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.689656] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.689677] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.689695] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.689715] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.689741] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.689766] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.689789] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.689793] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.689813] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.689815] [] ? __wake_up+0x13/0x20 [2660372.689835] [] ? ptlrpc_server_handle_req_in+0x8df/0xd60 [ptlrpc] [2660372.689855] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.689874] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.689876] [] kthread+0xd1/0xe0 [2660372.689878] [] ? insert_kthread_work+0x40/0x40 [2660372.689879] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.689881] [] ? insert_kthread_work+0x40/0x40 [2660372.689895] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2660372.693551] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [ll_ost00_074:24745] [2660372.693573] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.693593] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.693595] CPU: 16 PID: 24745 Comm: ll_ost00_074 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.693595] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.693596] task: ffff9c1438eb3150 ti: ffff9c1438c48000 task.ti: ffff9c1438c48000 [2660372.693599] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.693599] RSP: 0018:ffff9c1438c4bb68 EFLAGS: 00000246 [2660372.693600] RAX: 0000000000000000 RBX: ffff9c1438c4bb18 RCX: 0000000000810000 [2660372.693600] RDX: ffff9c159f29b880 RSI: 0000000000210101 RDI: ffff9c2a4a5112dc [2660372.693601] RBP: ffff9c1438c4bb68 R08: ffff9c159f41b880 R09: 0000000000000000 [2660372.693602] R10: ffff9c147036d800 R11: ffff9c144a9ece00 R12: ffff9c0324901050 [2660372.693602] R13: ffff9c1469365400 R14: 00000000a39d6037 R15: 0000000000000000 [2660372.693603] FS: 0000000000000000(0000) GS:ffff9c159f400000(0000) knlGS:0000000000000000 [2660372.693604] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.693604] CR2: 000000c002217010 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.693605] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.693605] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.693606] PKRU: 00000000 [2660372.693606] Call Trace: [2660372.693609] [] queued_spin_lock_slowpath+0xb/0xf [2660372.693611] [] _raw_spin_lock+0x20/0x30 [2660372.693626] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.693641] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.693662] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.693679] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.693699] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.693724] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.693748] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.693771] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.693774] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.693794] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.693814] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.693816] [] ? __wake_up+0x13/0x20 [2660372.693836] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.693855] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.693857] [] kthread+0xd1/0xe0 [2660372.693858] [] ? insert_kthread_work+0x40/0x40 [2660372.693860] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.693861] [] ? insert_kthread_work+0x40/0x40 [2660372.693876] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.697541] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ll_ost01_041:23881] [2660372.697563] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.697584] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.697586] CPU: 17 PID: 23881 Comm: ll_ost01_041 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.697586] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.697587] task: ffff9c2c947a8000 ti: ffff9c1543264000 task.ti: ffff9c1543264000 [2660372.697590] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.697591] RSP: 0018:ffff9c1543267b68 EFLAGS: 00000246 [2660372.697591] RAX: 0000000000000000 RBX: ffff9c1543267b18 RCX: 0000000000890000 [2660372.697592] RDX: ffff9c159f45b880 RSI: 0000000000910101 RDI: ffff9c2a4a5112dc [2660372.697592] RBP: ffff9c1543267b68 R08: ffff9c2d9d81b880 R09: 0000000000000000 [2660372.697593] R10: ffff9c147036d800 R11: ffff9c2d9b1cc440 R12: ffff9c2095999050 [2660372.697593] R13: ffff9c2ab62d9800 R14: 00000000c2afd02f R15: 0000000000000000 [2660372.697594] FS: 0000000000000000(0000) GS:ffff9c2d9d800000(0000) knlGS:0000000000000000 [2660372.697595] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.697596] CR2: 000000c0020ef010 CR3: 000000190da10000 CR4: 00000000007607e0 [2660372.697596] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.697597] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.697597] PKRU: 00000000 [2660372.697598] Call Trace: [2660372.697601] [] queued_spin_lock_slowpath+0xb/0xf [2660372.697602] [] _raw_spin_lock+0x20/0x30 [2660372.697621] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.697636] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.697657] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.697674] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.697695] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.697720] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.697744] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.697766] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.697771] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.697791] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.697811] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.697813] [] ? __wake_up+0x13/0x20 [2660372.697833] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.697853] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.697855] [] kthread+0xd1/0xe0 [2660372.697856] [] ? insert_kthread_work+0x40/0x40 [2660372.697858] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.697860] [] ? insert_kthread_work+0x40/0x40 [2660372.697874] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.701531] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [ll_ost00_110:26116] [2660372.701553] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.701573] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.701575] CPU: 18 PID: 26116 Comm: ll_ost00_110 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.701576] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.701576] task: ffff9c151a7b8000 ti: ffff9c142db54000 task.ti: ffff9c142db54000 [2660372.701579] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.701580] RSP: 0018:ffff9c142db57b68 EFLAGS: 00000246 [2660372.701581] RAX: 0000000000000000 RBX: ffff9c142db57b18 RCX: 0000000000910000 [2660372.701581] RDX: ffff9c2d9d85b880 RSI: 0000000000990101 RDI: ffff9c2a4a5112dc [2660372.701582] RBP: ffff9c142db57b68 R08: ffff9c159f45b880 R09: 0000000000000000 [2660372.701583] R10: ffff9c147036d800 R11: ffff9c144eb7d980 R12: ffff9c0324906850 [2660372.701583] R13: ffff9c123ae2e000 R14: 00000000a53c61d5 R15: 0000000000000000 [2660372.701584] FS: 0000000000000000(0000) GS:ffff9c159f440000(0000) knlGS:0000000000000000 [2660372.701585] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.701585] CR2: 000000c0046f1000 CR3: 000000172e636000 CR4: 00000000007607e0 [2660372.701586] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.701587] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.701587] PKRU: 00000000 [2660372.701587] Call Trace: [2660372.701590] [] queued_spin_lock_slowpath+0xb/0xf [2660372.701592] [] _raw_spin_lock+0x20/0x30 [2660372.701607] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.701622] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.701642] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.701660] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.701680] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.701705] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.701730] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.701752] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.701756] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.701776] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.701796] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.701798] [] ? __wake_up+0x13/0x20 [2660372.701818] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.701838] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.701839] [] kthread+0xd1/0xe0 [2660372.701841] [] ? insert_kthread_work+0x40/0x40 [2660372.701842] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.701844] [] ? insert_kthread_work+0x40/0x40 [2660372.701858] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660372.705522] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ll_ost01_114:24783] [2660372.705545] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660372.705566] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660372.705568] CPU: 19 PID: 24783 Comm: ll_ost01_114 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660372.705568] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660372.705569] task: ffff9c143c43e2a0 ti: ffff9c143e56c000 task.ti: ffff9c143e56c000 [2660372.705572] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660372.705573] RSP: 0018:ffff9c143e56fb68 EFLAGS: 00000246 [2660372.705573] RAX: 0000000000000000 RBX: ffff9c143e56fb18 RCX: 0000000000990000 [2660372.705574] RDX: ffff9c159f31b880 RSI: 0000000000410101 RDI: ffff9c2a4a5112dc [2660372.705574] RBP: ffff9c143e56fb68 R08: ffff9c2d9d85b880 R09: 0000000000000000 [2660372.705575] R10: ffff9c147036d800 R11: ffff9c14498adbc0 R12: ffff9c282abac050 [2660372.705575] R13: ffff9c20ee670c00 R14: 000000004be36980 R15: 0000000000000000 [2660372.705576] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2660372.705577] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660372.705578] CR2: 000000c00210b000 CR3: 000000190da10000 CR4: 00000000007607e0 [2660372.705578] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660372.705579] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660372.705579] PKRU: 00000000 [2660372.705579] Call Trace: [2660372.705583] [] queued_spin_lock_slowpath+0xb/0xf [2660372.705585] [] _raw_spin_lock+0x20/0x30 [2660372.705604] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660372.705618] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660372.705639] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660372.705657] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660372.705677] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660372.705703] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660372.705727] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660372.705750] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660372.705754] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660372.705774] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660372.705794] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660372.705796] [] ? __wake_up+0x13/0x20 [2660372.705816] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660372.705836] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660372.705838] [] kthread+0xd1/0xe0 [2660372.705839] [] ? insert_kthread_work+0x40/0x40 [2660372.705841] [] ret_from_fork_nospec_begin+0x7/0x21 [2660372.705842] [] ? insert_kthread_work+0x40/0x40 [2660372.705856] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660381.328325] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2660381.405634] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2660381.478621] CPU: 1 PID: 24744 Comm: ll_ost01_093 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2660381.492031] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2660381.500451] task: ffff9c1438eb20e0 ti: ffff9c1438e1c000 task.ti: ffff9c1438e1c000 [2660381.508785] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2660381.519584] RSP: 0018:ffff9c1438e1fb68 EFLAGS: 00000246 [2660381.525774] RAX: 0000000000000000 RBX: ffff9c1438e1fb18 RCX: 0000000000090000 [2660381.533780] RDX: ffff9c2d9d6db880 RSI: 0000000000390101 RDI: ffff9c2a4a5112dc [2660381.541784] RBP: ffff9c1438e1fb68 R08: ffff9c2d9d61b880 R09: 0000000000000000 [2660381.549792] R10: ffff9c147036d800 R11: ffff9c144c6a9d80 R12: ffff9c2acb070050 [2660381.557807] R13: ffff9c2b0e590800 R14: 00000000b3c87d6d R15: 0000000000000000 [2660381.565821] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2660381.574796] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2660381.581436] CR2: 00007f77d9493000 CR3: 000000190da10000 CR4: 00000000007607e0 [2660381.589466] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2660381.597490] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2660381.605512] PKRU: 00000000 [2660381.609126] Call Trace: [2660381.612476] [] queued_spin_lock_slowpath+0xb/0xf [2660381.619632] [] _raw_spin_lock+0x20/0x30 [2660381.626029] [] lock_res_and_lock+0x2c/0x50 [ptlrpc] [2660381.633466] [] ldlm_lock_enqueue+0x1b1/0xa20 [ptlrpc] [2660381.641080] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [2660381.648684] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2660381.656639] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2660381.665034] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2660381.672044] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2660381.679832] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2660381.688291] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2660381.696246] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2660381.704815] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2660381.712496] [] ? __wake_up+0x13/0x20 [2660381.718640] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2660381.725814] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2660381.734082] [] kthread+0xd1/0xe0 [2660381.739824] [] ? insert_kthread_work+0x40/0x40 [2660381.746765] [] ret_from_fork_nospec_begin+0x7/0x21 [2660381.754037] [] ? insert_kthread_work+0x40/0x40 [2660381.760953] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2660385.365818] Lustre: f2-OST0038: Client f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) reconnecting [2660385.365819] Lustre: f2-OST0014: Client f2-MDT0003-mdtlov_UUID (at 10.10.33.9@o2ib2) reconnecting [2660385.365836] Lustre: f2-OST0038: Connection restored to f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) [2660385.365836] Lustre: f2-OST0014: Connection restored to f2-MDT0003-mdtlov_UUID (at 10.10.33.9@o2ib2) [2660385.365837] Lustre: Skipped 15 previous similar messages [2660385.365837] Lustre: Skipped 15 previous similar messages [2660402.800901] Lustre: f2-OST0014: Client f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) reconnecting [2660402.800938] Lustre: f2-OST0038: Connection restored to f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) [2660402.800939] Lustre: Skipped 6 previous similar messages [2660402.801066] Lustre: f2-OST0038: deleting orphan objects from 0x0:6902131 to 0x0:6902218 [2660402.835103] Lustre: Skipped 7 previous similar messages [2660402.841519] Lustre: f2-OST0014: deleting orphan objects from 0x0:6903462 to 0x0:6903553 [2660403.528105] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000402:3500367 to 0x7c0000402:3500545 [2660403.543756] Lustre: f2-OST0038: deleting orphan objects from 0x940000402:3498623 to 0x940000402:3498785 [2660404.827033] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000400:5469847 to 0x7c0000400:5469876 [2660404.843475] Lustre: f2-OST0038: deleting orphan objects from 0x940000400:5471347 to 0x940000400:5471377 [2660406.583714] Lustre: f2-OST0014: Client f2-MDT0001-mdtlov_UUID (at 10.10.33.7@o2ib2) reconnecting [2660406.583745] Lustre: f2-OST0038: Connection restored to f2-MDT0001-mdtlov_UUID (at 10.10.33.7@o2ib2) [2660406.583747] Lustre: Skipped 5 previous similar messages [2660406.583892] Lustre: f2-OST0038: deleting orphan objects from 0x940000401:4569379 to 0x940000401:4569441 [2660406.619139] Lustre: Skipped 5 previous similar messages [2660406.626843] Lustre: f2-OST0014: deleting orphan objects from 0x7c0000401:4567490 to 0x7c0000401:4567649 [2660429.763000] Lustre: f2-OST0014: Client 58cca99f-ed47-1d9a-da31-abe8127d574e (at 10.10.32.232@o2ib2) reconnecting [2660429.763034] Lustre: f2-OST0038: Connection restored to 58cca99f-ed47-1d9a-da31-abe8127d574e (at 10.10.32.232@o2ib2) [2660429.763035] Lustre: Skipped 1 previous similar message [2660429.790970] Lustre: Skipped 1 previous similar message [2661616.408799] Lustre: f2-OST0014: Connection restored to f17c80d8-120f-0710-9d23-c665eb479bfc (at 10.10.33.22@o2ib2) [2661616.419957] Lustre: Skipped 4 previous similar messages [2661616.434544] Lustre: Mounted f2-client [2661620.745773] Lustre: Unmounted f2-client [2663449.146872] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ll_ost01_054:24606] [2663449.220693] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ldlm_cn01_051:73465] [2663449.155601] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic [2663449.220694] Modules linked in: [2663449.220694] lustre(OE) [2663449.220695] lmv(OE) [2663449.220695] mdc(OE) [2663449.220696] osc(OE) [2663449.220696] lov(OE) [2663449.220697] osp(OE) [2663449.220697] ofd(OE) [2663449.220698] lfsck(OE) [2663449.220698] ost(OE) [2663449.220699] mgc(OE) [2663449.220699] osd_zfs(OE) [2663449.220700] lquota(OE) [2663449.220700] fid(OE) [2663449.220701] fld(OE) [2663449.220701] ptlrpc(OE) [2663449.220702] obdclass(OE) [2663449.220702] ko2iblnd(OE) [2663449.220702] dm_round_robin [2663449.220703] dm_multipath [2663449.220703] lnet(OE) [2663449.220704] libcfs(OE) [2663449.220704] xt_comment [2663449.220705] xt_multiport [2663449.220705] zfs(POE) [2663449.220706] zunicode(POE) [2663449.220706] zlua(POE) [2663449.220707] zcommon(POE) [2663449.220707] znvpair(POE) [2663449.220708] zavl(POE) [2663449.220708] icp(POE) [2663449.220709] spl(OE) [2663449.220709] sd_mod [2663449.220709] sg [2663449.220710] nfsv3 [2663449.220710] nfs_acl [2663449.220711] mpt2sas [2663449.220711] raid_class [2663449.220712] scsi_transport_sas [2663449.220712] mptctl [2663449.220713] mptbase [2663449.220713] ip6t_REJECT [2663449.220713] nf_reject_ipv6 [2663449.220714] nf_conntrack_ipv6 [2663449.220714] nf_defrag_ipv6 [2663449.220714] ip6table_filter [2663449.220715] ip6_tables [2663449.220715] ipt_REJECT [2663449.220716] nf_reject_ipv4 [2663449.220716] nf_conntrack_ipv4 [2663449.220717] nf_defrag_ipv4 [2663449.220717] xt_conntrack [2663449.220717] nf_conntrack [2663449.220718] libcrc32c [2663449.220718] iptable_filter [2663449.220718] dell_rbu [2663449.220719] ib_isert [2663449.220719] iscsi_target_mod [2663449.220720] target_core_mod [2663449.220720] crc_t10dif [2663449.220720] crct10dif_generic [2663449.220721] ib_srp [2663449.220721] scsi_transport_srp [2663449.220722] scsi_tgt [2663449.220723] rpcrdma [2663449.220723] rdma_ucm [2663449.220724] ib_iser [2663449.220724] ib_umad [2663449.220724] rdma_cm [2663449.220725] ib_ipoib [2663449.220726] iw_cm [2663449.220726] libiscsi [2663449.220727] scsi_transport_iscsi [2663449.220727] ib_cm [2663449.220727] mlx5_ib [2663449.220728] ib_uverbs [2663449.220728] ib_core [2663449.220729] dm_mirror [2663449.220729] dm_region_hash [2663449.220729] dm_log [2663449.220730] dm_mod [2663449.220730] skx_edac [2663449.220731] intel_powerclamp [2663449.220731] coretemp [2663449.220732] intel_rapl [2663449.220732] iosf_mbi [2663449.220733] kvm_intel [2663449.220733] kvm [2663449.220734] irqbypass [2663449.220734] iTCO_wdt [2663449.220734] crct10dif_pclmul [2663449.220735] iTCO_vendor_support [2663449.220736] crct10dif_common [2663449.220736] crc32_pclmul [2663449.220737] dell_smbios [2663449.220737] ghash_clmulni_intel [2663449.220737] dell_wmi_descriptor [2663449.220738] dcdbas [2663449.220739] aesni_intel [2663449.220739] lrw [2663449.220740] gf128mul [2663449.220740] glue_helper [2663449.220740] ablk_helper [2663449.220741] cryptd [2663449.220742] pcspkr [2663449.220742] mgag200 [2663449.220742] ttm [2663449.220743] drm_kms_helper [2663449.220744] syscopyarea [2663449.220744] sysfillrect [2663449.220745] sysimgblt [2663449.220745] fb_sys_fops [2663449.220746] drm [2663449.220747] drm_panel_orientation_quirks [2663449.220747] lpc_ich [2663449.220748] i2c_i801 [2663449.220748] ahci [2663449.220748] libahci [2663449.220749] libata [2663449.220749] mei_me [2663449.220750] mei [2663449.220750] wmi [2663449.220751] ipmi_si [2663449.220751] ipmi_devintf [2663449.220752] ipmi_msghandler [2663449.220752] nfit [2663449.220753] libnvdimm [2663449.220753] acpi_power_meter [2663449.220754] acpi_pad [2663449.220755] pcc_cpufreq [2663449.220755] binfmt_misc [2663449.220756] ip_tables [2663449.220756] rpcsec_gss_krb5 [2663449.220757] auth_rpcgss [2663449.220757] nfsv4 [2663449.220758] dns_resolver [2663449.220758] nfs [2663449.220759] lockd [2663449.220759] grace [2663449.220760] fscache [2663449.220760] mlx5_core [2663449.220761] igb [2663449.220762] crc32c_intel [2663449.220762] mlxfw [2663449.220762] i2c_algo_bit [2663449.220763] devlink [2663449.220763] dca [2663449.220764] ptp [2663449.220764] pps_core [2663449.220764] sunrpc [2663449.220765] [2663449.220767] CPU: 19 PID: 73465 Comm: ldlm_cn01_051 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663449.220768] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663449.220769] task: ffff9c2c6e1ac1c0 ti: ffff9c146edfc000 task.ti: ffff9c146edfc000 [2663449.220770] RIP: 0010:[] [2663449.220807] [] ldlm_extent_policy.isra.13+0x224/0x810 [ptlrpc] [2663449.220808] RSP: 0018:ffff9c146edffac0 EFLAGS: 00000206 [2663449.220809] RAX: ffff9c071dc71f80 RBX: ffff9c146edffa70 RCX: 000000000000000a [2663449.220809] RDX: ffff9c0efbdd5820 RSI: 000000000480e000 RDI: ffff9c14505f8400 [2663449.220810] RBP: ffff9c146edffb48 R08: ffff9c2a4a5112f0 R09: 000000000480efff [2663449.220810] R10: 000000000480e000 R11: 000000000480e000 R12: ffff9c1ae1c1db80 [2663449.220811] R13: ffff9c146edffb48 R14: ffffbf0299acf5c0 R15: ffff9c1ae1c1db80 [2663449.220812] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2663449.220813] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663449.220813] CR2: 000000c002357000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663449.220814] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663449.220815] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663449.220815] PKRU: 00000000 [2663449.220815] Call Trace: [2663449.220834] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663449.220837] [] ? __slab_free+0x9d/0x290 [2663449.220853] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663449.220867] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663449.220877] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2663449.220891] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663449.220906] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663449.220923] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663449.220940] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663449.220956] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663449.220979] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663449.220999] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663449.221001] [] ? __wake_up+0x13/0x20 [2663449.221021] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663449.221041] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663449.221044] [] kthread+0xd1/0xe0 [2663449.221045] [] ? insert_kthread_work+0x40/0x40 [2663449.221048] [] ret_from_fork_nospec_begin+0x7/0x21 [2663449.221050] [] ? insert_kthread_work+0x40/0x40 [2663449.221050] Code: [2663449.221051] 39 [2663449.221051] d0 [2663449.221051] 48 [2663449.221052] 8d [2663449.221052] 42 [2663449.221052] a0 [2663449.221052] 0f [2663449.221053] 84 [2663449.221053] b8 [2663449.221053] 04 [2663449.221053] 00 [2663449.221054] 00 [2663449.221054] 48 [2663449.221054] 8b [2663449.221054] 75 [2663449.221055] b8 [2663449.221055] 31 [2663449.221055] c9 [2663449.221055] eb [2663449.221056] 24 [2663449.221056] 0f [2663449.221056] 1f [2663449.221056] 00 [2663449.221057] 49 [2663449.221057] 8b [2663449.221057] bf [2663449.221057] b8 [2663449.221058] 00 [2663449.221058] 00 [2663449.221058] 00 [2663449.221059] 48 [2663449.221059] 39 [2663449.221059] b8 [2663449.221059] b8 [2663449.221060] 00 [2663449.221060] 00 [2663449.221060] 00 [2663449.221060] 74 [2663449.221061] 2f [2663449.221061] 48 [2663449.221061] 8b [2663449.221061] 50 [2663449.221062] 60 [2663449.221062] <49> [2663449.221062] 39 [2663449.221062] d0 [2663449.221063] 48 [2663449.221063] 8d [2663449.221063] 42 [2663449.221063] a0 [2663449.221064] 0f [2663449.221064] 84 [2663449.221064] c7 [2663449.221064] 00 [2663449.221065] 00 [2663449.221065] 00 [2663449.221065] 48 [2663449.221065] 39 [2663449.221066] f3 [2663449.221066] 0f [2663449.221066] 84 [2663449.221066] de [2663449.221067] 02 [2663449.221067] 00 [2663449.221067] [2663449.685754] ib_srp [2663449.688712] scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace [2663449.763434] fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663449.772688] CPU: 1 PID: 24606 Comm: ll_ost01_054 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663449.786069] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663449.794482] task: ffff9c1599918000 ti: ffff9c1424f3c000 task.ti: ffff9c1424f3c000 [2663449.802809] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2663449.813590] RSP: 0018:ffff9c1424f3fad0 EFLAGS: 00000202 [2663449.819754] RAX: 0000000000000001 RBX: ffff9c2a63c47050 RCX: 0000000000000001 [2663449.827734] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2663449.835715] RBP: ffff9c1424f3fad0 R08: 0000000000000101 R09: 0000000000000158 [2663449.843702] R10: ffff9bff3fc07500 R11: ffff9c144aacea00 R12: ffffffffc14b9237 [2663449.851690] R13: ffff9c1424f3fa58 R14: 0000000000000003 R15: 0000000000000003 [2663449.859672] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2663449.868602] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663449.875197] CR2: 00007f77d9493000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663449.883189] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663449.891175] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663449.899154] PKRU: 00000000 [2663449.902727] Call Trace: [2663449.906034] [] queued_spin_lock_slowpath+0xb/0xf [2663449.913148] [] _raw_spin_lock+0x20/0x30 [2663449.919492] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663449.926809] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663449.934364] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663449.942358] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663449.949593] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663449.957520] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663449.965887] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663449.972874] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663449.980643] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663449.989071] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663449.997008] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663450.005545] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663450.013195] [] ? __wake_up+0x13/0x20 [2663450.019321] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663450.026478] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663450.034732] [] kthread+0xd1/0xe0 [2663450.040480] [] ? insert_kthread_work+0x40/0x40 [2663450.047430] [] ret_from_fork_nospec_begin+0x7/0x21 [2663450.054709] [] ? insert_kthread_work+0x40/0x40 [2663450.061634] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2663453.162104] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ll_ost01_083:24719] [2663453.170893] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663453.248461] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663453.321788] CPU: 7 PID: 24719 Comm: ll_ost01_083 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663453.335300] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663453.343782] task: ffff9c148f0ce2a0 ti: ffff9c1430674000 task.ti: ffff9c1430674000 [2663453.352167] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2663453.363017] RSP: 0018:ffff9c1430677ad0 EFLAGS: 00000202 [2663453.369242] RAX: 0000000000000101 RBX: ffff9c2c7aba4050 RCX: 0000000000390000 [2663453.377298] RDX: 0000000000390101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2663453.385358] RBP: ffff9c1430677ad0 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2663453.393414] R10: ffff9bff3fc07500 R11: ffff9c2c589bca80 R12: ffffffffc14b9237 [2663453.401481] R13: ffff9c1430677a58 R14: 0000000000000003 R15: 0000000000000003 [2663453.409547] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2663453.418566] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663453.425247] CR2: 00007f76f4e82000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663453.433323] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663453.441391] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663453.449455] PKRU: 00000000 [2663453.453104] Call Trace: [2663453.456491] [] queued_spin_lock_slowpath+0xb/0xf [2663453.463691] [] _raw_spin_lock+0x20/0x30 [2663453.470115] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663453.477510] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663453.485149] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663453.493214] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663453.500516] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663453.508516] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663453.516949] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663453.524003] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663453.531824] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663453.540323] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663453.548327] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663453.556938] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663453.564645] [] ? __wake_up+0x13/0x20 [2663453.570805] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663453.577997] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663453.586277] [] kthread+0xd1/0xe0 [2663453.592035] [] ? insert_kthread_work+0x40/0x40 [2663453.598996] [] ret_from_fork_nospec_begin+0x7/0x21 [2663453.606281] [] ? insert_kthread_work+0x40/0x40 [2663453.613206] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2663477.078919] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ll_ost01_054:24606] [2663477.152740] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ldlm_cn01_051:73465] [2663477.087707] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic [2663477.152740] Modules linked in: [2663477.152741] lustre(OE) [2663477.152742] lmv(OE) [2663477.152742] mdc(OE) [2663477.152743] osc(OE) [2663477.152743] lov(OE) [2663477.152744] osp(OE) [2663477.152744] ofd(OE) [2663477.152744] lfsck(OE) [2663477.152745] ost(OE) [2663477.152745] mgc(OE) [2663477.152746] osd_zfs(OE) [2663477.152746] lquota(OE) [2663477.152746] fid(OE) [2663477.152747] fld(OE) [2663477.152747] ptlrpc(OE) [2663477.152748] obdclass(OE) [2663477.152748] ko2iblnd(OE) [2663477.152749] dm_round_robin [2663477.152749] dm_multipath [2663477.152750] lnet(OE) [2663477.152750] libcfs(OE) [2663477.152751] xt_comment [2663477.152751] xt_multiport [2663477.152751] zfs(POE) [2663477.152752] zunicode(POE) [2663477.152752] zlua(POE) [2663477.152753] zcommon(POE) [2663477.152753] znvpair(POE) [2663477.152753] zavl(POE) [2663477.152754] icp(POE) [2663477.152754] spl(OE) [2663477.152755] sd_mod [2663477.152755] sg [2663477.152756] nfsv3 [2663477.152756] nfs_acl [2663477.152757] mpt2sas [2663477.152757] raid_class [2663477.152758] scsi_transport_sas [2663477.152758] mptctl [2663477.152758] mptbase [2663477.152759] ip6t_REJECT [2663477.152759] nf_reject_ipv6 [2663477.152760] nf_conntrack_ipv6 [2663477.152760] nf_defrag_ipv6 [2663477.152761] ip6table_filter [2663477.152761] ip6_tables [2663477.152761] ipt_REJECT [2663477.152762] nf_reject_ipv4 [2663477.152762] nf_conntrack_ipv4 [2663477.152763] nf_defrag_ipv4 [2663477.152763] xt_conntrack [2663477.152764] nf_conntrack [2663477.152764] libcrc32c [2663477.152765] iptable_filter [2663477.152765] dell_rbu [2663477.152765] ib_isert [2663477.152766] iscsi_target_mod [2663477.152766] target_core_mod [2663477.152767] crc_t10dif [2663477.152767] crct10dif_generic [2663477.152767] ib_srp [2663477.152768] scsi_transport_srp [2663477.152768] scsi_tgt [2663477.152769] rpcrdma [2663477.152769] rdma_ucm [2663477.152770] ib_iser [2663477.152770] ib_umad [2663477.152771] rdma_cm [2663477.152772] ib_ipoib [2663477.152772] iw_cm [2663477.152773] libiscsi [2663477.152773] scsi_transport_iscsi [2663477.152773] ib_cm [2663477.152774] mlx5_ib [2663477.152774] ib_uverbs [2663477.152775] ib_core [2663477.152775] dm_mirror [2663477.152776] dm_region_hash [2663477.152776] dm_log [2663477.152776] dm_mod [2663477.152777] skx_edac [2663477.152777] intel_powerclamp [2663477.152778] coretemp [2663477.152778] intel_rapl [2663477.152779] iosf_mbi [2663477.152779] kvm_intel [2663477.152780] kvm [2663477.152780] irqbypass [2663477.152781] iTCO_wdt [2663477.152781] crct10dif_pclmul [2663477.152782] iTCO_vendor_support [2663477.152782] crct10dif_common [2663477.152783] crc32_pclmul [2663477.152783] dell_smbios [2663477.152784] ghash_clmulni_intel [2663477.152784] dell_wmi_descriptor [2663477.152785] dcdbas [2663477.152785] aesni_intel [2663477.152786] lrw [2663477.152786] gf128mul [2663477.152787] glue_helper [2663477.152787] ablk_helper [2663477.152788] cryptd [2663477.152788] pcspkr [2663477.152789] mgag200 [2663477.152789] ttm [2663477.152790] drm_kms_helper [2663477.152791] syscopyarea [2663477.152791] sysfillrect [2663477.152792] sysimgblt [2663477.152792] fb_sys_fops [2663477.152793] drm [2663477.152793] drm_panel_orientation_quirks [2663477.152794] lpc_ich [2663477.152794] i2c_i801 [2663477.152795] ahci [2663477.152795] libahci [2663477.152796] libata [2663477.152796] mei_me [2663477.152797] mei [2663477.152797] wmi [2663477.152798] ipmi_si [2663477.152798] ipmi_devintf [2663477.152799] ipmi_msghandler [2663477.152799] nfit [2663477.152799] libnvdimm [2663477.152800] acpi_power_meter [2663477.152801] acpi_pad [2663477.152801] pcc_cpufreq [2663477.152802] binfmt_misc [2663477.152802] ip_tables [2663477.152803] rpcsec_gss_krb5 [2663477.152803] auth_rpcgss [2663477.152804] nfsv4 [2663477.152805] dns_resolver [2663477.152805] nfs [2663477.152806] lockd [2663477.152806] grace [2663477.152807] fscache [2663477.152807] mlx5_core [2663477.152807] igb [2663477.152808] crc32c_intel [2663477.152809] mlxfw [2663477.152809] i2c_algo_bit [2663477.152809] devlink [2663477.152810] dca [2663477.152810] ptp [2663477.152811] pps_core [2663477.152811] sunrpc [2663477.152812] [2663477.152814] CPU: 19 PID: 73465 Comm: ldlm_cn01_051 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663477.152814] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663477.152815] task: ffff9c2c6e1ac1c0 ti: ffff9c146edfc000 task.ti: ffff9c146edfc000 [2663477.152816] RIP: 0010:[] [2663477.152850] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663477.152851] RSP: 0018:ffff9c146edffac0 EFLAGS: 00000287 [2663477.152851] RAX: ffff9c0def27af40 RBX: ffff9c146edffa70 RCX: 0000000000000015 [2663477.152852] RDX: 0000000000000004 RSI: 000000000480e000 RDI: ffff9c1467d33000 [2663477.152852] RBP: ffff9c146edffb48 R08: ffff9c2a4a5112f0 R09: 00000000047e5fff [2663477.152853] R10: 000000000480e000 R11: 000000000480e000 R12: ffff9c2ad3683380 [2663477.152854] R13: 000000000480afff R14: ffffffffffffffff R15: ffff9c2ad3683380 [2663477.152855] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2663477.152855] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663477.152856] CR2: 000000c002357000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663477.152857] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663477.152857] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663477.152858] PKRU: 00000000 [2663477.152858] Call Trace: [2663477.152876] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663477.152892] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663477.152907] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663477.152917] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2663477.152931] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663477.152946] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663477.152963] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663477.152979] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663477.152995] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663477.153017] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663477.153037] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663477.153041] [] ? __wake_up+0x13/0x20 [2663477.153061] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663477.153081] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663477.153083] [] kthread+0xd1/0xe0 [2663477.153085] [] ? insert_kthread_work+0x40/0x40 [2663477.153087] [] ret_from_fork_nospec_begin+0x7/0x21 [2663477.153089] [] ? insert_kthread_work+0x40/0x40 [2663477.153090] Code: [2663477.153090] b8 [2663477.153090] 00 [2663477.153091] 00 [2663477.153091] 00 [2663477.153091] 74 [2663477.153091] 2f [2663477.153092] 48 [2663477.153092] 8b [2663477.153092] 50 [2663477.153092] 60 [2663477.153093] 49 [2663477.153093] 39 [2663477.153093] d0 [2663477.153093] 48 [2663477.153094] 8d [2663477.153094] 42 [2663477.153094] a0 [2663477.153094] 0f [2663477.153095] 84 [2663477.153095] c7 [2663477.153095] 00 [2663477.153095] 00 [2663477.153096] 00 [2663477.153096] 48 [2663477.153096] 39 [2663477.153096] f3 [2663477.153097] 0f [2663477.153097] 84 [2663477.153097] de [2663477.153097] 02 [2663477.153098] 00 [2663477.153098] 00 [2663477.153098] 49 [2663477.153098] 39 [2663477.153099] c7 [2663477.153099] 74 [2663477.153099] e1 [2663477.153100] 8b [2663477.153100] 90 [2663477.153100] 98 [2663477.153100] 00 [2663477.153101] 00 [2663477.153101] 00 [2663477.153101] <44> [2663477.153101] 85 [2663477.153102] 24 [2663477.153102] 95 [2663477.153102] 00 [2663477.153102] 01 [2663477.153103] 5a [2663477.153103] c1 [2663477.153103] 75 [2663477.153103] c1 [2663477.153104] 83 [2663477.153104] c1 [2663477.153104] 01 [2663477.153104] 48 [2663477.153105] 89 [2663477.153105] f7 [2663477.153105] 48 [2663477.153105] 8b [2663477.153106] 90 [2663477.153106] d0 [2663477.153106] 00 [2663477.153106] [2663477.610554] ib_srp [2663477.613519] scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace [2663477.688294] fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663477.697564] CPU: 1 PID: 24606 Comm: ll_ost01_054 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663477.710973] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663477.719387] task: ffff9c1599918000 ti: ffff9c1424f3c000 task.ti: ffff9c1424f3c000 [2663477.727711] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2663477.738491] RSP: 0018:ffff9c1424f3fad0 EFLAGS: 00000202 [2663477.744649] RAX: 0000000000000001 RBX: ffff9c2a63c47050 RCX: 0000000000000001 [2663477.752630] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2663477.760600] RBP: ffff9c1424f3fad0 R08: 0000000000000101 R09: 0000000000000158 [2663477.768570] R10: ffff9bff3fc07500 R11: ffff9c144aacea00 R12: ffffffffc14b9237 [2663477.776550] R13: ffff9c1424f3fa58 R14: 0000000000000003 R15: 0000000000000003 [2663477.784532] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2663477.793471] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663477.800059] CR2: 00007f77d9493000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663477.808040] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663477.816018] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663477.823997] PKRU: 00000000 [2663477.827561] Call Trace: [2663477.830877] [] queued_spin_lock_slowpath+0xb/0xf [2663477.837999] [] _raw_spin_lock+0x20/0x30 [2663477.844346] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663477.851659] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663477.859215] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663477.867200] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663477.874427] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663477.882362] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663477.890731] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663477.897714] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663477.905474] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663477.913906] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663477.921838] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663477.930379] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663477.938028] [] ? __wake_up+0x13/0x20 [2663477.944144] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663477.951305] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663477.959566] [] kthread+0xd1/0xe0 [2663477.965316] [] ? insert_kthread_work+0x40/0x40 [2663477.972264] [] ret_from_fork_nospec_begin+0x7/0x21 [2663477.979552] [] ? insert_kthread_work+0x40/0x40 [2663477.986486] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2663481.094150] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ll_ost01_083:24719] [2663481.102929] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663481.180480] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663481.253772] CPU: 7 PID: 24719 Comm: ll_ost01_083 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663481.267275] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663481.275746] task: ffff9c148f0ce2a0 ti: ffff9c1430674000 task.ti: ffff9c1430674000 [2663481.284133] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2663481.294985] RSP: 0018:ffff9c1430677ad0 EFLAGS: 00000202 [2663481.301218] RAX: 0000000000000101 RBX: ffff9c2c7aba4050 RCX: 0000000000390000 [2663481.309277] RDX: 0000000000390101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2663481.317332] RBP: ffff9c1430677ad0 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2663481.325391] R10: ffff9bff3fc07500 R11: ffff9c2c589bca80 R12: ffffffffc14b9237 [2663481.333458] R13: ffff9c1430677a58 R14: 0000000000000003 R15: 0000000000000003 [2663481.341523] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2663481.350539] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663481.357223] CR2: 00007f76f4e82000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663481.365299] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663481.373373] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663481.381441] PKRU: 00000000 [2663481.385089] Call Trace: [2663481.388472] [] queued_spin_lock_slowpath+0xb/0xf [2663481.395663] [] _raw_spin_lock+0x20/0x30 [2663481.402089] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663481.409488] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663481.417124] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663481.425189] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663481.432491] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663481.440487] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663481.448924] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663481.455975] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663481.463799] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663481.472297] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663481.480304] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663481.488914] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663481.496621] [] ? __wake_up+0x13/0x20 [2663481.502787] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663481.509982] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663481.518270] [] kthread+0xd1/0xe0 [2663481.524028] [] ? insert_kthread_work+0x40/0x40 [2663481.530986] [] ret_from_fork_nospec_begin+0x7/0x21 [2663481.538276] [] ? insert_kthread_work+0x40/0x40 [2663481.545205] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2663481.735589] INFO: rcu_sched self-detected stall on CPU [2663481.735593] INFO: rcu_sched self-detected stall on CPU [2663481.735593] { [2663481.735593] 19 [2663481.735594] } [2663481.735603] (t=60000 jiffies g=209315053 c=209315052 q=40906) [2663481.735604] Task dump for CPU 1: [2663481.735605] ll_ost01_054 R [2663481.735605] running task [2663481.735606] 0 24606 2 0x00000008 [2663481.735607] Call Trace: [2663481.735650] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663481.735673] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663481.735682] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663481.735703] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663481.735723] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663481.735727] [] ? __wake_up+0x13/0x20 [2663481.735747] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663481.735766] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663481.735768] [] kthread+0xd1/0xe0 [2663481.735770] [] ? insert_kthread_work+0x40/0x40 [2663481.735771] [] ret_from_fork_nospec_begin+0x7/0x21 [2663481.735773] [] ? insert_kthread_work+0x40/0x40 [2663481.735774] Task dump for CPU 19: [2663481.735774] ldlm_cn01_051 R [2663481.735775] running task [2663481.735775] 0 73465 2 0x00000008 [2663481.735775] Call Trace: [2663481.735776] [2663481.735778] [] sched_show_task+0xa8/0x110 [2663481.735780] [] dump_cpu_task+0x39/0x70 [2663481.735782] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663481.735784] [] rcu_check_callbacks+0x442/0x730 [2663481.735787] [] ? tick_sched_do_timer+0x50/0x50 [2663481.735789] [] update_process_times+0x46/0x80 [2663481.735790] [] tick_sched_handle+0x30/0x70 [2663481.735792] [] tick_sched_timer+0x39/0x80 [2663481.735794] [] __hrtimer_run_queues+0x10e/0x270 [2663481.735796] [] hrtimer_interrupt+0xaf/0x1d0 [2663481.735798] [] local_apic_timer_interrupt+0x3b/0x60 [2663481.735800] [] smp_apic_timer_interrupt+0x43/0x60 [2663481.735802] [] apic_timer_interrupt+0x16a/0x170 [2663481.735802] [2663481.735818] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663481.735834] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663481.735849] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663481.735864] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663481.735869] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2663481.735884] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663481.735898] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663481.735914] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663481.735931] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663481.735947] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663481.735967] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663481.735985] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663481.735987] [] ? __wake_up+0x13/0x20 [2663481.736006] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663481.736025] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663481.736027] [] kthread+0xd1/0xe0 [2663481.736028] [] ? insert_kthread_work+0x40/0x40 [2663481.736030] [] ret_from_fork_nospec_begin+0x7/0x21 [2663481.736031] [] ? insert_kthread_work+0x40/0x40 [2663482.098987] { [2663482.101329] 1} (t=60367 jiffies g=209315053 c=209315052 q=40907) [2663482.106750] Task dump for CPU 1: [2663482.110633] ll_ost01_054 R running task 0 24606 2 0x00000008 [2663482.118378] Call Trace: [2663482.121484] [] sched_show_task+0xa8/0x110 [2663482.128436] [] dump_cpu_task+0x39/0x70 [2663482.134498] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663482.141076] [] rcu_check_callbacks+0x442/0x730 [2663482.147830] [] ? tick_sched_do_timer+0x50/0x50 [2663482.154579] [] update_process_times+0x46/0x80 [2663482.161248] [] tick_sched_handle+0x30/0x70 [2663482.167646] [] tick_sched_timer+0x39/0x80 [2663482.173956] [] __hrtimer_run_queues+0x10e/0x270 [2663482.180788] [] hrtimer_interrupt+0xaf/0x1d0 [2663482.187297] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663482.195423] [] local_apic_timer_interrupt+0x3b/0x60 [2663482.202600] [] smp_apic_timer_interrupt+0x43/0x60 [2663482.209601] [] apic_timer_interrupt+0x16a/0x170 [2663482.216429] [] ? native_queued_spin_lock_slowpath+0x1ce/0x200 [2663482.225114] [] queued_spin_lock_slowpath+0xb/0xf [2663482.232047] [] _raw_spin_lock+0x20/0x30 [2663482.238207] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663482.245350] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663482.252735] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663482.260550] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663482.267606] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663482.275364] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663482.283556] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663482.290359] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663482.297940] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663482.306181] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663482.313931] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663482.322292] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663482.329759] [] ? __wake_up+0x13/0x20 [2663482.335692] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663482.342673] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663482.350751] [] kthread+0xd1/0xe0 [2663482.356319] [] ? insert_kthread_work+0x40/0x40 [2663482.363090] [] ret_from_fork_nospec_begin+0x7/0x21 [2663482.370213] [] ? insert_kthread_work+0x40/0x40 [2663482.376989] Task dump for CPU 19: [2663482.380991] ldlm_cn01_051 R running task 0 73465 2 0x00000008 [2663482.388772] Call Trace: [2663482.391937] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663482.399619] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663482.407382] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663482.415925] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663482.423400] [] ? __wake_up+0x13/0x20 [2663482.429347] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663482.436499] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663482.444574] [] ? kthread+0xd1/0xe0 [2663482.450314] [] ? insert_kthread_work+0x40/0x40 [2663482.457085] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663482.464373] [] ? insert_kthread_work+0x40/0x40 [2663509.001262] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ll_ost01_054:24606] [2663509.026196] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ll_ost01_083:24719] [2663509.009909] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) [2663509.026197] Modules linked in: [2663509.026198] lustre(OE) [2663509.026198] lmv(OE) [2663509.026199] mdc(OE) [2663509.026199] osc(OE) [2663509.026200] lov(OE) [2663509.026200] osp(OE) [2663509.026200] ofd(OE) [2663509.026201] lfsck(OE) [2663509.026201] ost(OE) [2663509.026201] mgc(OE) [2663509.026202] osd_zfs(OE) [2663509.026202] lquota(OE) [2663509.026203] fid(OE) [2663509.026203] fld(OE) [2663509.026203] ptlrpc(OE) [2663509.026204] obdclass(OE) [2663509.026204] ko2iblnd(OE) [2663509.026205] dm_round_robin [2663509.026205] dm_multipath [2663509.026206] lnet(OE) [2663509.026206] libcfs(OE) [2663509.026206] xt_comment [2663509.026207] xt_multiport [2663509.026207] zfs(POE) [2663509.026208] zunicode(POE) [2663509.026208] zlua(POE) [2663509.026209] zcommon(POE) [2663509.026209] znvpair(POE) [2663509.026210] zavl(POE) [2663509.026210] icp(POE) [2663509.026211] spl(OE) [2663509.026211] sd_mod [2663509.026212] sg [2663509.026212] nfsv3 [2663509.026213] nfs_acl [2663509.026213] mpt2sas [2663509.026213] raid_class [2663509.026214] scsi_transport_sas [2663509.026214] mptctl [2663509.026215] mptbase [2663509.026215] ip6t_REJECT [2663509.026216] nf_reject_ipv6 [2663509.026217] nf_conntrack_ipv6 [2663509.026217] nf_defrag_ipv6 [2663509.026217] ip6table_filter [2663509.026218] ip6_tables [2663509.026218] ipt_REJECT [2663509.026219] nf_reject_ipv4 [2663509.026219] nf_conntrack_ipv4 [2663509.026219] nf_defrag_ipv4 [2663509.026220] xt_conntrack [2663509.026220] nf_conntrack [2663509.026221] libcrc32c [2663509.026221] iptable_filter [2663509.026222] dell_rbu [2663509.026222] ib_isert [2663509.026223] iscsi_target_mod [2663509.026223] target_core_mod [2663509.026223] crc_t10dif [2663509.026224] crct10dif_generic [2663509.026224] ib_srp [2663509.026225] scsi_transport_srp [2663509.026225] scsi_tgt [2663509.026226] rpcrdma [2663509.026226] rdma_ucm [2663509.026227] ib_iser [2663509.026227] ib_umad [2663509.026228] rdma_cm [2663509.026228] ib_ipoib [2663509.026228] iw_cm [2663509.026229] libiscsi [2663509.026229] scsi_transport_iscsi [2663509.026230] ib_cm [2663509.026230] mlx5_ib [2663509.026230] ib_uverbs [2663509.026231] ib_core [2663509.026231] dm_mirror [2663509.026232] dm_region_hash [2663509.026232] dm_log [2663509.026232] dm_mod [2663509.026233] skx_edac [2663509.026233] intel_powerclamp [2663509.026234] coretemp [2663509.026234] intel_rapl [2663509.026235] iosf_mbi [2663509.026235] kvm_intel [2663509.026235] kvm [2663509.026236] irqbypass [2663509.026236] iTCO_wdt [2663509.026236] crct10dif_pclmul [2663509.026237] iTCO_vendor_support [2663509.026237] crct10dif_common [2663509.026237] crc32_pclmul [2663509.026238] dell_smbios [2663509.026238] ghash_clmulni_intel [2663509.026239] dell_wmi_descriptor [2663509.026239] dcdbas [2663509.026240] aesni_intel [2663509.026240] lrw [2663509.026240] gf128mul [2663509.026241] glue_helper [2663509.026241] ablk_helper [2663509.026242] cryptd [2663509.026242] pcspkr [2663509.026243] mgag200 [2663509.026243] ttm [2663509.026244] drm_kms_helper [2663509.026244] syscopyarea [2663509.026245] sysfillrect [2663509.026245] sysimgblt [2663509.026246] fb_sys_fops [2663509.026246] drm [2663509.026247] drm_panel_orientation_quirks [2663509.026247] lpc_ich [2663509.026248] i2c_i801 [2663509.026249] ahci [2663509.026249] libahci [2663509.026249] libata [2663509.026250] mei_me [2663509.026250] mei [2663509.026251] wmi [2663509.026251] ipmi_si [2663509.026251] ipmi_devintf [2663509.026252] ipmi_msghandler [2663509.026252] nfit [2663509.026253] libnvdimm [2663509.026254] acpi_power_meter [2663509.026254] acpi_pad [2663509.026255] pcc_cpufreq [2663509.026255] binfmt_misc [2663509.026256] ip_tables [2663509.026256] rpcsec_gss_krb5 [2663509.026257] auth_rpcgss [2663509.026257] nfsv4 [2663509.026257] dns_resolver [2663509.026258] nfs [2663509.026258] lockd [2663509.026258] grace [2663509.026259] fscache [2663509.026259] mlx5_core [2663509.026259] igb [2663509.026260] crc32c_intel [2663509.026260] mlxfw [2663509.026261] i2c_algo_bit [2663509.026261] devlink [2663509.026261] dca [2663509.026262] ptp [2663509.026262] pps_core [2663509.026263] sunrpc [2663509.026263] [2663509.026266] CPU: 7 PID: 24719 Comm: ll_ost01_083 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663509.026266] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663509.026267] task: ffff9c148f0ce2a0 ti: ffff9c1430674000 task.ti: ffff9c1430674000 [2663509.026268] RIP: 0010:[] [2663509.026272] [] native_queued_spin_lock_slowpath+0x156/0x200 [2663509.026273] RSP: 0018:ffff9c1430677ad0 EFLAGS: 00000202 [2663509.026273] RAX: 0000000000000101 RBX: ffff9c2c7aba4050 RCX: 0000000000390000 [2663509.026274] RDX: 0000000000390101 RSI: 0000000000000101 RDI: ffff9c2a4a5112dc [2663509.026275] RBP: ffff9c1430677ad0 R08: ffff9c2d9d6db880 R09: 0000000000000000 [2663509.026275] R10: ffff9bff3fc07500 R11: ffff9c2c589bca80 R12: ffffffffc14b9237 [2663509.026276] R13: ffff9c1430677a58 R14: 0000000000000003 R15: 0000000000000003 [2663509.026277] FS: 0000000000000000(0000) GS:ffff9c2d9d6c0000(0000) knlGS:0000000000000000 [2663509.026278] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663509.026278] CR2: 00007f76f4e82000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663509.026279] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663509.026280] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663509.026280] PKRU: 00000000 [2663509.026280] Call Trace: [2663509.026285] [] queued_spin_lock_slowpath+0xb/0xf [2663509.026288] [] _raw_spin_lock+0x20/0x30 [2663509.026299] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663509.026331] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663509.026341] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663509.026345] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663509.026364] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663509.026385] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663509.026413] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663509.026437] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663509.026459] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663509.026463] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663509.026483] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663509.026502] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663509.026505] [] ? __wake_up+0x13/0x20 [2663509.026525] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663509.026544] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663509.026546] [] kthread+0xd1/0xe0 [2663509.026548] [] ? insert_kthread_work+0x40/0x40 [2663509.026550] [] ret_from_fork_nospec_begin+0x7/0x21 [2663509.026551] [] ? insert_kthread_work+0x40/0x40 [2663509.026552] Code: [2663509.026553] 8b [2663509.026553] 08 [2663509.026553] 4d [2663509.026553] 85 [2663509.026554] c9 [2663509.026554] 74 [2663509.026554] 04 [2663509.026554] 41 [2663509.026555] 0f [2663509.026555] 18 [2663509.026555] 09 [2663509.026555] 8b [2663509.026556] 17 [2663509.026556] 0f [2663509.026556] b7 [2663509.026556] c2 [2663509.026557] 85 [2663509.026557] c0 [2663509.026557] 74 [2663509.026557] 21 [2663509.026558] 83 [2663509.026558] f8 [2663509.026558] 03 [2663509.026558] 75 [2663509.026559] 10 [2663509.026559] eb [2663509.026559] 1a [2663509.026559] 66 [2663509.026560] 2e [2663509.026560] 0f [2663509.026560] 1f [2663509.026560] 84 [2663509.026561] 00 [2663509.026561] 00 [2663509.026561] 00 [2663509.026561] 00 [2663509.026562] 00 [2663509.026562] 85 [2663509.026562] c0 [2663509.026562] 74 [2663509.026563] 0c [2663509.026563] f3 [2663509.026563] 90 [2663509.026564] <8b> [2663509.026564] 17 [2663509.026564] 0f [2663509.026564] b7 [2663509.026565] c2 [2663509.026565] 83 [2663509.026565] f8 [2663509.026565] 03 [2663509.026566] 75 [2663509.026566] f0 [2663509.026566] be [2663509.026566] 01 [2663509.026567] 00 [2663509.026567] 00 [2663509.026567] 00 [2663509.026567] eb [2663509.026568] 15 [2663509.026568] 66 [2663509.026568] 0f [2663509.026568] 1f [2663509.026569] 84 [2663509.026569] [2663509.075079] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ldlm_cn01_051:73465] [2663509.075079] Modules linked in: [2663509.075080] lustre(OE) [2663509.075080] lmv(OE) [2663509.075081] mdc(OE) [2663509.075081] osc(OE) [2663509.075082] lov(OE) [2663509.075082] osp(OE) [2663509.075083] ofd(OE) [2663509.075083] lfsck(OE) [2663509.075084] ost(OE) [2663509.075084] mgc(OE) [2663509.075085] osd_zfs(OE) [2663509.075085] lquota(OE) [2663509.075085] fid(OE) [2663509.075086] fld(OE) [2663509.075086] ptlrpc(OE) [2663509.075087] obdclass(OE) [2663509.075087] ko2iblnd(OE) [2663509.075088] dm_round_robin [2663509.075088] dm_multipath [2663509.075089] lnet(OE) [2663509.075089] libcfs(OE) [2663509.075089] xt_comment [2663509.075090] xt_multiport [2663509.075090] zfs(POE) [2663509.075091] zunicode(POE) [2663509.075091] zlua(POE) [2663509.075092] zcommon(POE) [2663509.075092] znvpair(POE) [2663509.075092] zavl(POE) [2663509.075093] icp(POE) [2663509.075093] spl(OE) [2663509.075094] sd_mod [2663509.075094] sg [2663509.075094] nfsv3 [2663509.075095] nfs_acl [2663509.075095] mpt2sas [2663509.075096] raid_class [2663509.075097] scsi_transport_sas [2663509.075097] mptctl [2663509.075097] mptbase [2663509.075098] ip6t_REJECT [2663509.075099] nf_reject_ipv6 [2663509.075099] nf_conntrack_ipv6 [2663509.075100] nf_defrag_ipv6 [2663509.075100] ip6table_filter [2663509.075101] ip6_tables [2663509.075101] ipt_REJECT [2663509.075101] nf_reject_ipv4 [2663509.075102] nf_conntrack_ipv4 [2663509.075102] nf_defrag_ipv4 [2663509.075103] xt_conntrack [2663509.075104] nf_conntrack [2663509.075104] libcrc32c [2663509.075104] iptable_filter [2663509.075105] dell_rbu [2663509.075105] ib_isert [2663509.075106] iscsi_target_mod [2663509.075106] target_core_mod [2663509.075107] crc_t10dif [2663509.075107] crct10dif_generic [2663509.075108] ib_srp [2663509.075108] scsi_transport_srp [2663509.075108] scsi_tgt [2663509.075109] rpcrdma [2663509.075109] rdma_ucm [2663509.075110] ib_iser [2663509.075110] ib_umad [2663509.075110] rdma_cm [2663509.075111] ib_ipoib [2663509.075111] iw_cm [2663509.075112] libiscsi [2663509.075112] scsi_transport_iscsi [2663509.075112] ib_cm [2663509.075113] mlx5_ib [2663509.075113] ib_uverbs [2663509.075113] ib_core [2663509.075114] dm_mirror [2663509.075114] dm_region_hash [2663509.075115] dm_log [2663509.075115] dm_mod [2663509.075115] skx_edac [2663509.075116] intel_powerclamp [2663509.075116] coretemp [2663509.075117] intel_rapl [2663509.075117] iosf_mbi [2663509.075117] kvm_intel [2663509.075118] kvm [2663509.075118] irqbypass [2663509.075119] iTCO_wdt [2663509.075119] crct10dif_pclmul [2663509.075120] iTCO_vendor_support [2663509.075120] crct10dif_common [2663509.075120] crc32_pclmul [2663509.075121] dell_smbios [2663509.075121] ghash_clmulni_intel [2663509.075122] dell_wmi_descriptor [2663509.075122] dcdbas [2663509.075122] aesni_intel [2663509.075123] lrw [2663509.075123] gf128mul [2663509.075124] glue_helper [2663509.075124] ablk_helper [2663509.075124] cryptd [2663509.075125] pcspkr [2663509.075125] mgag200 [2663509.075126] ttm [2663509.075126] drm_kms_helper [2663509.075126] syscopyarea [2663509.075127] sysfillrect [2663509.075127] sysimgblt [2663509.075128] fb_sys_fops [2663509.075128] drm [2663509.075129] drm_panel_orientation_quirks [2663509.075129] lpc_ich [2663509.075130] i2c_i801 [2663509.075130] ahci [2663509.075130] libahci [2663509.075131] libata [2663509.075131] mei_me [2663509.075132] mei [2663509.075132] wmi [2663509.075132] ipmi_si [2663509.075133] ipmi_devintf [2663509.075133] ipmi_msghandler [2663509.075134] nfit [2663509.075134] libnvdimm [2663509.075135] acpi_power_meter [2663509.075135] acpi_pad [2663509.075136] pcc_cpufreq [2663509.075136] binfmt_misc [2663509.075137] ip_tables [2663509.075137] rpcsec_gss_krb5 [2663509.075137] auth_rpcgss [2663509.075138] nfsv4 [2663509.075138] dns_resolver [2663509.075139] nfs [2663509.075139] lockd [2663509.075139] grace [2663509.075140] fscache [2663509.075140] mlx5_core [2663509.075141] igb [2663509.075141] crc32c_intel [2663509.075141] mlxfw [2663509.075142] i2c_algo_bit [2663509.075142] devlink [2663509.075143] dca [2663509.075143] ptp [2663509.075143] pps_core [2663509.075144] sunrpc [2663509.075144] [2663509.075146] CPU: 19 PID: 73465 Comm: ldlm_cn01_051 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663509.075147] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663509.075148] task: ffff9c2c6e1ac1c0 ti: ffff9c146edfc000 task.ti: ffff9c146edfc000 [2663509.075148] RIP: 0010:[] [2663509.075174] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663509.075175] RSP: 0018:ffff9c146edffac0 EFLAGS: 00000287 [2663509.075176] RAX: ffff9c145473d580 RBX: ffff9c146edffa70 RCX: 0000000000000017 [2663509.075176] RDX: 0000000000000004 RSI: 0000000004b20000 RDI: ffff9c145c7bc800 [2663509.075177] RBP: ffff9c146edffb48 R08: ffff9c2a4a5112f0 R09: 0000000004b20fff [2663509.075178] R10: 0000000004b20000 R11: 0000000004b20000 R12: ffff9c1ebb21dc80 [2663509.075178] R13: ffff9c146edffac8 R14: ffffffffffffff10 R15: ffff9c1ebb21dc80 [2663509.075179] FS: 0000000000000000(0000) GS:ffff9c2d9d840000(0000) knlGS:0000000000000000 [2663509.075180] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663509.075181] CR2: 000000c002357000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663509.075181] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663509.075182] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663509.075182] PKRU: 00000000 [2663509.075183] Call Trace: [2663509.075200] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663509.075216] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663509.075231] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663509.075238] [] ? cfs_hash_multi_bd_unlock+0x68/0x90 [libcfs] [2663509.075252] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663509.075267] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663509.075284] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663509.075301] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663509.075317] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663509.075338] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663509.075358] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663509.075361] [] ? __wake_up+0x13/0x20 [2663509.075381] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663509.075401] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663509.075403] [] kthread+0xd1/0xe0 [2663509.075405] [] ? insert_kthread_work+0x40/0x40 [2663509.075407] [] ret_from_fork_nospec_begin+0x7/0x21 [2663509.075408] [] ? insert_kthread_work+0x40/0x40 [2663509.075409] Code: [2663509.075409] b8 [2663509.075410] 00 [2663509.075410] 00 [2663509.075410] 00 [2663509.075411] 74 [2663509.075411] 2f [2663509.075411] 48 [2663509.075411] 8b [2663509.075412] 50 [2663509.075412] 60 [2663509.075412] 49 [2663509.075412] 39 [2663509.075413] d0 [2663509.075413] 48 [2663509.075413] 8d [2663509.075413] 42 [2663509.075414] a0 [2663509.075414] 0f [2663509.075414] 84 [2663509.075414] c7 [2663509.075415] 00 [2663509.075415] 00 [2663509.075415] 00 [2663509.075415] 48 [2663509.075416] 39 [2663509.075416] f3 [2663509.075416] 0f [2663509.075416] 84 [2663509.075417] de [2663509.075417] 02 [2663509.075417] 00 [2663509.075417] 00 [2663509.075418] 49 [2663509.075418] 39 [2663509.075418] c7 [2663509.075418] 74 [2663509.075419] e1 [2663509.075419] 8b [2663509.075419] 90 [2663509.075419] 98 [2663509.075420] 00 [2663509.075420] 00 [2663509.075420] 00 [2663509.075421] <44> [2663509.075421] 85 [2663509.075421] 24 [2663509.075421] 95 [2663509.075422] 00 [2663509.075422] 01 [2663509.075422] 5a [2663509.075422] c1 [2663509.075423] 75 [2663509.075423] c1 [2663509.075423] 83 [2663509.075423] c1 [2663509.075424] 01 [2663509.075424] 48 [2663509.075424] 89 [2663509.075424] f7 [2663509.075425] 48 [2663509.075425] 8b [2663509.075425] 90 [2663509.075425] d0 [2663509.075426] 00 [2663509.075426] [2663509.952427] ko2iblnd(OE) [2663509.955918] dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul [2663510.031879] iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663510.088864] CPU: 1 PID: 24606 Comm: ll_ost01_054 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663510.102307] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663510.110763] task: ffff9c1599918000 ti: ffff9c1424f3c000 task.ti: ffff9c1424f3c000 [2663510.119131] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x1ce/0x200 [2663510.129975] RSP: 0018:ffff9c1424f3fad0 EFLAGS: 00000202 [2663510.136188] RAX: 0000000000000001 RBX: ffff9c2a63c47050 RCX: 0000000000000001 [2663510.144214] RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9c2a4a5112dc [2663510.152252] RBP: ffff9c1424f3fad0 R08: 0000000000000101 R09: 0000000000000158 [2663510.160285] R10: ffff9bff3fc07500 R11: ffff9c144aacea00 R12: ffffffffc14b9237 [2663510.168309] R13: ffff9c1424f3fa58 R14: 0000000000000003 R15: 0000000000000003 [2663510.176349] FS: 0000000000000000(0000) GS:ffff9c2d9d600000(0000) knlGS:0000000000000000 [2663510.185343] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663510.191998] CR2: 00007f77d9493000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663510.200045] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663510.208080] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663510.216120] PKRU: 00000000 [2663510.219742] Call Trace: [2663510.223108] [] queued_spin_lock_slowpath+0xb/0xf [2663510.230284] [] _raw_spin_lock+0x20/0x30 [2663510.236671] [] ofd_intent_policy+0x1d9/0x920 [ofd] [2663510.244054] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [2663510.251657] [] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs] [2663510.259691] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [2663510.266985] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [2663510.274965] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663510.283383] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663510.290428] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663510.298232] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663510.306699] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663510.314677] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663510.323278] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663510.330981] [] ? __wake_up+0x13/0x20 [2663510.337148] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663510.344337] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663510.352619] [] kthread+0xd1/0xe0 [2663510.358377] [] ? insert_kthread_work+0x40/0x40 [2663510.365346] [] ret_from_fork_nospec_begin+0x7/0x21 [2663510.372632] [] ? insert_kthread_work+0x40/0x40 [2663510.379548] Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 [2663830.553984] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2663830.565034] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.162@o2ib2 (5): c: 6, oc: 0, rc: 8 [2663831.551556] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds [2663831.562535] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 4 previous similar messages [2663831.573491] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.146@o2ib2 (6): c: 7, oc: 0, rc: 8 [2663831.586933] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 4 previous similar messages [2663836.495538] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.152@o2ib2 added to recovery queue. Health = 900 [2663836.495542] LustreError: 25910:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(982468) req@ffff9c27173b7850 x1655452809189424/t0(0) o4->be6f45a8-ee72-f37d-b7dd-e314b3427b46@937@gni5:573/0 lens 608/448 e 0 to 0 dl 1585780678 ref 1 fl Interpret:/0/0 rc 0/0 [2663836.495544] LustreError: 25910:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 17 previous similar messages [2663836.495613] Lustre: f2-OST0038: Bulk IO write error with be6f45a8-ee72-f37d-b7dd-e314b3427b46 (at 937@gni5), client will retry: rc = -110 [2663836.560687] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 7 previous similar messages [2663839.532143] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds [2663839.543171] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 3 previous similar messages [2663839.554187] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.33.44@o2ib2 (105): c: 8, oc: 0, rc: 8 [2663839.567871] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 3 previous similar messages [2663841.527290] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds [2663841.538333] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 14 previous similar messages [2663841.549461] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.33.18@o2ib2 (107): c: 8, oc: 0, rc: 8 [2663841.563221] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 14 previous similar messages [2663841.574039] LustreError: 38525:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(271640) req@ffff9c2a54cf6850 x1655457684866864/t0(0) o4->67982e77-aac1-86f4-2232-572510a8411b@304@gni5:575/0 lens 608/448 e 0 to 0 dl 1585780680 ref 1 fl Interpret:/0/0 rc 0/0 [2663841.574062] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.150@o2ib2 added to recovery queue. Health = 900 [2663841.574063] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 4 previous similar messages [2663841.574096] Lustre: f2-OST0038: Bulk IO write error with bc9df5eb-7268-fea8-f7e9-735749d53066 (at 1055@gni5), client will retry: rc = -110 [2663841.574097] Lustre: Skipped 6 previous similar messages [2663841.647590] LustreError: 38525:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 11 previous similar messages [2663846.515156] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.146@o2ib2: 16 seconds [2663846.526547] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 2 previous similar messages [2663846.536888] LNetError: 18738:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.10.33.33@o2ib2 added to recovery queue. Health = 900 [2663846.561057] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.165@o2ib2 added to recovery queue. Health = 900 [2663846.576036] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 2 previous similar messages [2663846.587911] LustreError: 26083:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(1048576) req@ffff9c1de5aae850 x1661790537921264/t0(0) o4->86ea0768-4cc3-adbd-0fea-96124af095ba@1810@gni4:581/0 lens 608/448 e 0 to 0 dl 1585780686 ref 1 fl Interpret:/0/0 rc 0/0 [2663847.512730] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.146@o2ib2: 17 seconds [2663847.524168] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 8 previous similar messages [2663848.510308] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.146@o2ib2: 18 seconds [2663848.521732] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 8 previous similar messages [2663850.505444] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.146@o2ib2: 20 seconds [2663850.516859] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 8 previous similar messages [2663851.503015] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2663851.514248] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 2 previous similar messages [2663851.525466] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.227@o2ib2 (57): c: 7, oc: 0, rc: 8 [2663851.539536] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 2 previous similar messages [2663851.574848] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.152@o2ib2 added to recovery queue. Health = 900 [2663851.574867] Lustre: f2-OST0014: Bulk IO write error with c45139db-fa92-af86-b554-a5472cd7a2ee (at 287@gni5), client will retry: rc = -110 [2663851.574869] Lustre: Skipped 6 previous similar messages [2663852.167393] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ll_ost00_056:24709] [2663852.171383] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ll_ost00_033:24598] [2663852.171420] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.171454] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.171457] CPU: 2 PID: 24598 Comm: ll_ost00_033 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.171457] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.171458] task: ffff9c2c71fe41c0 ti: ffff9c146b1cc000 task.ti: ffff9c146b1cc000 [2663852.171465] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.171466] RSP: 0018:ffff9c146b1cfb90 EFLAGS: 00000246 [2663852.171467] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000110000 [2663852.171467] RDX: ffff9c159f3db880 RSI: 0000000000710000 RDI: ffff9c2a4a5112dc [2663852.171468] RBP: ffff9c146b1cfb90 R08: ffff9c159f25b880 R09: 0000000000000000 [2663852.171469] R10: ffff9c146aa90500 R11: ffff9c141bec8000 R12: 0000000000710000 [2663852.171469] R13: ffff9c159f3db880 R14: 0000000000110000 R15: 0000000000000000 [2663852.171470] FS: 0000000000000000(0000) GS:ffff9c159f240000(0000) knlGS:0000000000000000 [2663852.171471] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.171472] CR2: 000000c000ae5000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.171472] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.171473] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.171473] PKRU: 00000000 [2663852.171474] Call Trace: [2663852.171482] [] queued_spin_lock_slowpath+0xb/0xf [2663852.171486] [] _raw_spin_lock+0x20/0x30 [2663852.171537] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.171552] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.171573] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.171596] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.171628] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.171653] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.171676] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.171683] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.171703] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.171723] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.171742] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663852.171761] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663852.171780] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.171800] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.171804] [] kthread+0xd1/0xe0 [2663852.171805] [] ? insert_kthread_work+0x40/0x40 [2663852.171808] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.171809] [] ? insert_kthread_work+0x40/0x40 [2663852.171823] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.178366] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ll_ost00_123:26911] [2663852.178392] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.178415] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.178417] CPU: 4 PID: 26911 Comm: ll_ost00_123 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.178418] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.178419] task: ffff9c1460079070 ti: ffff9c1445fd8000 task.ti: ffff9c1445fd8000 [2663852.178423] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.178423] RSP: 0018:ffff9c1445fdbb90 EFLAGS: 00000246 [2663852.178424] RAX: 0000000000000000 RBX: ffff9c1445fdbadc RCX: 0000000000210000 [2663852.178425] RDX: ffff9c159f35b880 RSI: 0000000000510001 RDI: ffff9c2a4a5112dc [2663852.178426] RBP: ffff9c1445fdbb90 R08: ffff9c159f29b880 R09: 0000000000000000 [2663852.178426] R10: ffff9c2b0e592100 R11: ffff9c0c5c0d0000 R12: 0000000000000002 [2663852.178427] R13: ffff9c065d579f80 R14: ffff9c1445fdbbd8 R15: 00000000047e5fff [2663852.178428] FS: 0000000000000000(0000) GS:ffff9c159f280000(0000) knlGS:0000000000000000 [2663852.178429] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.178429] CR2: 000000c00130d010 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.178430] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.178430] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.178431] PKRU: 00000000 [2663852.178431] Call Trace: [2663852.178435] [] queued_spin_lock_slowpath+0xb/0xf [2663852.178437] [] _raw_spin_lock+0x20/0x30 [2663852.178461] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.178476] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.178494] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.178515] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.178542] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.178567] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.178589] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.178594] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.178614] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.178634] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.178637] [] ? wake_up_state+0x20/0x20 [2663852.178656] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.178676] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.178678] [] kthread+0xd1/0xe0 [2663852.178679] [] ? insert_kthread_work+0x40/0x40 [2663852.178681] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.178682] [] ? insert_kthread_work+0x40/0x40 [2663852.178697] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.187345] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ll_ost00_131:26919] [2663852.187371] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.187398] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.187400] CPU: 6 PID: 26919 Comm: ll_ost00_131 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.187400] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.187401] task: ffff9c148ed2c1c0 ti: ffff9c1448394000 task.ti: ffff9c1448394000 [2663852.187405] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.187406] RSP: 0018:ffff9c1448397b90 EFLAGS: 00000246 [2663852.187407] RAX: 0000000000000000 RBX: ffff9c1448397adc RCX: 0000000000310000 [2663852.187407] RDX: ffff9c159f31b880 RSI: 0000000000410000 RDI: ffff9c2a4a5112dc [2663852.187408] RBP: ffff9c1448397b90 R08: ffff9c159f2db880 R09: 0000000000000000 [2663852.187408] R10: ffff9c133da57900 R11: ffff9bfefdbb0000 R12: 0000000000000002 [2663852.187409] R13: ffff9c023cf62d00 R14: ffff9c1448397bd8 R15: 00000000047e5fff [2663852.187410] FS: 0000000000000000(0000) GS:ffff9c159f2c0000(0000) knlGS:0000000000000000 [2663852.187411] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.187411] CR2: 00007fcb82c09000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663852.187412] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.187412] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.187413] PKRU: 00000000 [2663852.187413] Call Trace: [2663852.187417] [] queued_spin_lock_slowpath+0xb/0xf [2663852.187420] [] _raw_spin_lock+0x20/0x30 [2663852.187458] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.187473] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.187491] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.187513] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.187540] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.187564] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.187586] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.187592] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.187612] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.187631] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.187635] [] ? __wake_up+0x13/0x20 [2663852.187654] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.187674] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.187676] [] kthread+0xd1/0xe0 [2663852.187678] [] ? insert_kthread_work+0x40/0x40 [2663852.187679] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.187681] [] ? insert_kthread_work+0x40/0x40 [2663852.187695] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.195325] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ll_ost00_112:26900] [2663852.195347] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.195369] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.195371] CPU: 8 PID: 26900 Comm: ll_ost00_112 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.195372] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.195373] task: ffff9c14446341c0 ti: ffff9c1450cc4000 task.ti: ffff9c1450cc4000 [2663852.195376] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.195376] RSP: 0018:ffff9c1450cc7b90 EFLAGS: 00000246 [2663852.195377] RAX: 0000000000000000 RBX: ffff9c1450cc7adc RCX: 0000000000410000 [2663852.195377] RDX: ffff9c159f25b880 RSI: 0000000000110000 RDI: ffff9c2a4a5112dc [2663852.195378] RBP: ffff9c1450cc7b90 R08: ffff9c159f31b880 R09: 0000000000000000 [2663852.195378] R10: ffff9c146b4bc500 R11: ffff9c0dacf38000 R12: 0000000000000002 [2663852.195379] R13: ffff9c1429192640 R14: ffff9c1450cc7bd8 R15: 000000000480afff [2663852.195380] FS: 0000000000000000(0000) GS:ffff9c159f300000(0000) knlGS:0000000000000000 [2663852.195381] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.195381] CR2: 00007f9d0f438000 CR3: 0000002fa937e000 CR4: 00000000007607e0 [2663852.195382] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.195383] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.195383] PKRU: 00000000 [2663852.195383] Call Trace: [2663852.195387] [] queued_spin_lock_slowpath+0xb/0xf [2663852.195389] [] _raw_spin_lock+0x20/0x30 [2663852.195409] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.195424] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.195442] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.195463] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.195488] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.195513] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.195535] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.195540] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.195560] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.195579] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.195581] [] ? queued_spin_lock_slowpath+0xb/0xf [2663852.195583] [] ? _raw_spin_lock+0x20/0x30 [2663852.195602] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.195622] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.195623] [] kthread+0xd1/0xe0 [2663852.195625] [] ? insert_kthread_work+0x40/0x40 [2663852.195626] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.195628] [] ? insert_kthread_work+0x40/0x40 [2663852.195642] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.203305] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ll_ost00_070:24739] [2663852.203330] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.203353] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.203355] CPU: 10 PID: 24739 Comm: ll_ost00_070 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.203356] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.203357] task: ffff9c2c947820e0 ti: ffff9c1434898000 task.ti: ffff9c1434898000 [2663852.203360] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.203361] RSP: 0018:ffff9c143489bb90 EFLAGS: 00000246 [2663852.203362] RAX: 0000000000000000 RBX: ffff9c143489badc RCX: 0000000000510000 [2663852.203362] RDX: ffff9c159f4db880 RSI: 0000000000b10000 RDI: ffff9c2a4a5112dc [2663852.203363] RBP: ffff9c143489bb90 R08: ffff9c159f35b880 R09: 0000000000000000 [2663852.203363] R10: ffff9c123814ad00 R11: ffff9c0b5f498000 R12: 0000000000000002 [2663852.203364] R13: ffff9c0925ed21c0 R14: ffff9c143489bbd8 R15: 00000000047e5fff [2663852.203365] FS: 0000000000000000(0000) GS:ffff9c159f340000(0000) knlGS:0000000000000000 [2663852.203365] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.203366] CR2: 000000c0015ed000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.203367] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.203367] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.203368] PKRU: 00000000 [2663852.203368] Call Trace: [2663852.203372] [] queued_spin_lock_slowpath+0xb/0xf [2663852.203374] [] _raw_spin_lock+0x20/0x30 [2663852.203413] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.203428] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.203446] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.203467] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.203494] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.203518] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.203541] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.203546] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.203566] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.203586] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.203605] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663852.203607] [] ? queued_spin_lock_slowpath+0xb/0xf [2663852.203608] [] ? _raw_spin_lock+0x20/0x30 [2663852.203627] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.203647] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.203649] [] kthread+0xd1/0xe0 [2663852.203651] [] ? insert_kthread_work+0x40/0x40 [2663852.203653] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.203654] [] ? insert_kthread_work+0x40/0x40 [2663852.203669] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.211286] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ll_ost00_073:24742] [2663852.211309] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.211330] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.211332] CPU: 12 PID: 24742 Comm: ll_ost00_073 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.211332] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.211333] task: ffff9c1438eb0000 ti: ffff9c1438830000 task.ti: ffff9c1438830000 [2663852.211336] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.211337] RSP: 0018:ffff9c1438833b90 EFLAGS: 00000246 [2663852.211337] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000610000 [2663852.211338] RDX: ffff9c159f45b880 RSI: 0000000000910000 RDI: ffff9c2a4a5112dc [2663852.211338] RBP: ffff9c1438833b90 R08: ffff9c159f39b880 R09: 0000000000000000 [2663852.211339] R10: ffff9c146184bd00 R11: ffff9c0f017c0000 R12: 0000000000910000 [2663852.211339] R13: ffff9c159f45b880 R14: 0000000000610000 R15: 0000000000000000 [2663852.211340] FS: 0000000000000000(0000) GS:ffff9c159f380000(0000) knlGS:0000000000000000 [2663852.211341] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.211342] CR2: 000000c000f8d000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.211342] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.211343] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.211343] PKRU: 00000000 [2663852.211344] Call Trace: [2663852.211347] [] queued_spin_lock_slowpath+0xb/0xf [2663852.211349] [] _raw_spin_lock+0x20/0x30 [2663852.211370] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.211387] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.211405] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.211428] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.211456] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.211482] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.211507] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.211512] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.211534] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.211555] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.211557] [] ? queued_spin_lock_slowpath+0xb/0xf [2663852.211558] [] ? _raw_spin_lock+0x20/0x30 [2663852.211579] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.211599] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.211601] [] kthread+0xd1/0xe0 [2663852.211603] [] ? insert_kthread_work+0x40/0x40 [2663852.211604] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.211606] [] ? insert_kthread_work+0x40/0x40 [2663852.211620] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.219266] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [ll_ost00_089:24777] [2663852.219290] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.219313] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.219315] CPU: 14 PID: 24777 Comm: ll_ost00_089 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.219316] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.219317] task: ffff9c143c438000 ti: ffff9c143beb4000 task.ti: ffff9c143beb4000 [2663852.219320] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.219321] RSP: 0018:ffff9c143beb7b90 EFLAGS: 00000246 [2663852.219321] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000710000 [2663852.219322] RDX: ffff9c159f39b880 RSI: 0000000000610000 RDI: ffff9c2a4a5112dc [2663852.219322] RBP: ffff9c143beb7b90 R08: ffff9c159f3db880 R09: 0000000000000000 [2663852.219323] R10: ffff9c12cda40d00 R11: ffff9c07b8908000 R12: 0000000000610000 [2663852.219324] R13: ffff9c159f39b880 R14: 0000000000710000 R15: 0000000000000000 [2663852.219325] FS: 0000000000000000(0000) GS:ffff9c159f3c0000(0000) knlGS:0000000000000000 [2663852.219325] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.219326] CR2: 00007fcb82c09000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.219327] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.219327] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.219328] PKRU: 00000000 [2663852.219328] Call Trace: [2663852.219332] [] queued_spin_lock_slowpath+0xb/0xf [2663852.219334] [] _raw_spin_lock+0x20/0x30 [2663852.219372] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.219387] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.219405] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.219426] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.219453] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.219477] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.219500] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.219505] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.219525] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.219545] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.219564] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663852.219566] [] ? queued_spin_lock_slowpath+0xb/0xf [2663852.219567] [] ? _raw_spin_lock+0x20/0x30 [2663852.219586] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.219606] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.219608] [] kthread+0xd1/0xe0 [2663852.219609] [] ? insert_kthread_work+0x40/0x40 [2663852.219611] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.219612] [] ? insert_kthread_work+0x40/0x40 [2663852.219627] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.228247] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ll_ost00_092:24784] [2663852.228269] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.228290] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.228292] CPU: 16 PID: 24784 Comm: ll_ost00_092 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.228293] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.228294] task: ffff9c143d550000 ti: ffff9c1455194000 task.ti: ffff9c1455194000 [2663852.228324] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663852.228324] RSP: 0018:ffff9c1455197a38 EFLAGS: 00000283 [2663852.228325] RAX: ffff9c251cf2f500 RBX: ffff9c050ffb8000 RCX: 0000000000000006 [2663852.228326] RDX: 0000000000000004 RSI: 000000000480a000 RDI: ffff9c1451807800 [2663852.228326] RBP: ffff9c1455197ac0 R08: ffff9c2a4a5112f0 R09: 000000000480afff [2663852.228327] R10: 000000000480a000 R11: 000000000480a000 R12: ffff9c0cae3de510 [2663852.228327] R13: ffff9c050ffb8000 R14: ffffffffffffffff R15: ffffffff94223abd [2663852.228328] FS: 0000000000000000(0000) GS:ffff9c159f400000(0000) knlGS:0000000000000000 [2663852.228329] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.228330] CR2: 000000c000eff000 CR3: 0000001724304000 CR4: 00000000007607e0 [2663852.228330] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.228331] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.228331] PKRU: 00000000 [2663852.228331] Call Trace: [2663852.228349] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663852.228365] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663852.228380] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663852.228394] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663852.228409] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.228426] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.228446] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.228472] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.228496] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.228519] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.228523] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.228543] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.228563] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.228583] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.228602] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.228604] [] kthread+0xd1/0xe0 [2663852.228606] [] ? insert_kthread_work+0x40/0x40 [2663852.228608] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.228609] [] ? insert_kthread_work+0x40/0x40 [2663852.228623] Code: b8 00 00 00 74 2f 48 8b 50 60 49 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 00 49 39 c7 74 e1 8b 90 98 00 00 00 <44> 85 24 95 00 01 5a c1 75 c1 83 c1 01 48 89 f7 48 8b 90 d0 00 [2663852.236225] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ll_ost00_038:24607] [2663852.236250] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.236273] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.236275] CPU: 18 PID: 24607 Comm: ll_ost00_038 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.236275] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.236276] task: ffff9c1551e720e0 ti: ffff9c143e1ac000 task.ti: ffff9c143e1ac000 [2663852.236279] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.236280] RSP: 0018:ffff9c143e1afb90 EFLAGS: 00000246 [2663852.236280] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000910000 [2663852.236281] RDX: ffff9c2d9d8db880 RSI: 0000000000a10001 RDI: ffff9c2a4a5112dc [2663852.236282] RBP: ffff9c143e1afb90 R08: ffff9c159f45b880 R09: 0000000000000000 [2663852.236282] R10: ffff9c1f9b4d0500 R11: ffff9c00f5008000 R12: 0000000000a10000 [2663852.236283] R13: ffff9c159f49b880 R14: 0000000000910000 R15: 0000000000000000 [2663852.236284] FS: 0000000000000000(0000) GS:ffff9c159f440000(0000) knlGS:0000000000000000 [2663852.236284] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.236285] CR2: 000000c000df3010 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.236286] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.236286] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.236287] PKRU: 00000000 [2663852.236287] Call Trace: [2663852.236290] [] queued_spin_lock_slowpath+0xb/0xf [2663852.236292] [] _raw_spin_lock+0x20/0x30 [2663852.236331] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.236346] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.236364] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.236384] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.236412] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.236437] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.236459] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.236464] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.236484] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.236504] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.236507] [] ? __wake_up+0x13/0x20 [2663852.236526] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.236546] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.236548] [] kthread+0xd1/0xe0 [2663852.236549] [] ? insert_kthread_work+0x40/0x40 [2663852.236551] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.236552] [] ? insert_kthread_work+0x40/0x40 [2663852.236567] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.244206] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_ost00_049:24692] [2663852.244228] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.244249] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.244250] CPU: 20 PID: 24692 Comm: ll_ost00_049 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.244251] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.244252] task: ffff9c152a34b150 ti: ffff9c1443e64000 task.ti: ffff9c1443e64000 [2663852.244255] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.244256] RSP: 0018:ffff9c1443e67b90 EFLAGS: 00000246 [2663852.244256] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000a10000 [2663852.244257] RDX: ffff9c159f21b880 RSI: 0000000000010000 RDI: ffff9c2a4a5112dc [2663852.244257] RBP: ffff9c1443e67b90 R08: ffff9c159f49b880 R09: 0000000000000000 [2663852.244258] R10: ffff9c146a96e500 R11: ffff9c0785538000 R12: 0000000000b90000 [2663852.244258] R13: ffff9c2d9d8db880 R14: 0000000000a10000 R15: 0000000000000000 [2663852.244259] FS: 0000000000000000(0000) GS:ffff9c159f480000(0000) knlGS:0000000000000000 [2663852.244260] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.244261] CR2: 00007f77d9493000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.244261] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.244262] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.244262] PKRU: 00000000 [2663852.244263] Call Trace: [2663852.244266] [] queued_spin_lock_slowpath+0xb/0xf [2663852.244268] [] _raw_spin_lock+0x20/0x30 [2663852.244289] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.244304] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.244321] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.244343] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.244368] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.244393] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.244416] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.244420] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.244441] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.244462] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.244481] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663852.244483] [] ? queued_spin_lock_slowpath+0xb/0xf [2663852.244484] [] ? _raw_spin_lock+0x20/0x30 [2663852.244503] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.244523] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.244524] [] kthread+0xd1/0xe0 [2663852.244526] [] ? insert_kthread_work+0x40/0x40 [2663852.244528] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.244529] [] ? insert_kthread_work+0x40/0x40 [2663852.244543] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.252187] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ll_ost00_138:26926] [2663852.252210] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.252233] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.252235] CPU: 22 PID: 26926 Comm: ll_ost00_138 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.252235] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.252236] task: ffff9c1432feb150 ti: ffff9c1448310000 task.ti: ffff9c1448310000 [2663852.252239] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663852.252240] RSP: 0018:ffff9c1448313b90 EFLAGS: 00000246 [2663852.252240] RAX: 0000000000000000 RBX: ffff9c1448313adc RCX: 0000000000b10000 [2663852.252241] RDX: ffff9c159f2db880 RSI: 0000000000310000 RDI: ffff9c2a4a5112dc [2663852.252242] RBP: ffff9c1448313b90 R08: ffff9c159f4db880 R09: 0000000000000000 [2663852.252242] R10: ffff9c272c285900 R11: ffff9c06f6418000 R12: 0000000000000002 [2663852.252243] R13: ffff9c0ce138ad00 R14: ffff9c1448313bd8 R15: 000000000480efff [2663852.252244] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2663852.252244] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.252245] CR2: 000000c0013a7000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663852.252246] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.252246] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.252246] PKRU: 00000000 [2663852.252247] Call Trace: [2663852.252250] [] queued_spin_lock_slowpath+0xb/0xf [2663852.252252] [] _raw_spin_lock+0x20/0x30 [2663852.252290] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.252305] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.252322] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663852.252343] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663852.252371] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663852.252395] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663852.252418] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663852.252422] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663852.252443] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.252462] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.252464] [] ? queued_spin_lock_slowpath+0xb/0xf [2663852.252465] [] ? _raw_spin_lock+0x20/0x30 [2663852.252485] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.252505] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.252507] [] kthread+0xd1/0xe0 [2663852.252509] [] ? insert_kthread_work+0x40/0x40 [2663852.252510] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.252512] [] ? insert_kthread_work+0x40/0x40 [2663852.252526] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663852.256176] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ldlm_cn01_046:57960] [2663852.256206] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663852.256236] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663852.256239] CPU: 23 PID: 57960 Comm: ldlm_cn01_046 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663852.256239] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663852.256240] task: ffff9c152e665230 ti: ffff9c0f7f358000 task.ti: ffff9c0f7f358000 [2663852.256245] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [2663852.256246] RSP: 0018:ffff9c0f7f35bc18 EFLAGS: 00000246 [2663852.256246] RAX: 0000000000000000 RBX: ffff9c0f7f35bbc0 RCX: 0000000000b90000 [2663852.256247] RDX: ffff9c159f49b880 RSI: 0000000000a10000 RDI: ffff9c2a4a5112dc [2663852.256247] RBP: ffff9c0f7f35bc18 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2663852.256248] R10: 0000000000000001 R11: ffff9c2a27221400 R12: ffff9c2a27221400 [2663852.256248] R13: 0000000000000292 R14: 0000000000000000 R15: ffff9c2a27221400 [2663852.256249] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2663852.256250] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663852.256251] CR2: 00007f39ae98dbc0 CR3: 000000190da10000 CR4: 00000000007607e0 [2663852.256251] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663852.256252] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663852.256252] PKRU: 00000000 [2663852.256253] Call Trace: [2663852.256258] [] queued_spin_lock_slowpath+0xb/0xf [2663852.256261] [] _raw_spin_lock+0x20/0x30 [2663852.256291] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663852.256306] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663852.256324] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663852.256342] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663852.256359] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663852.256380] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663852.256401] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663852.256403] [] ? __wake_up+0x13/0x20 [2663852.256423] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663852.256443] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663852.256445] [] kthread+0xd1/0xe0 [2663852.256447] [] ? insert_kthread_work+0x40/0x40 [2663852.256449] [] ret_from_fork_nospec_begin+0x7/0x21 [2663852.256450] [] ? insert_kthread_work+0x40/0x40 [2663852.256464] Code: 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [2663856.597643] LustreError: 38525:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(554980) req@ffff9c2a2a400850 x1655456596290672/t0(0) o4->82715c6a-65b9-6395-a0a4-f7419fc57306@677@gni5:589/0 lens 608/448 e 0 to 0 dl 1585780694 ref 1 fl Interpret:/0/0 rc 0/0 [2663856.597645] LustreError: 38525:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 4 previous similar messages [2663857.689875] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663857.767495] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663857.840812] CPU: 0 PID: 24709 Comm: ll_ost00_056 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663857.854315] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663857.862788] task: ffff9c1595ed62a0 ti: ffff9c142f5c0000 task.ti: ffff9c142f5c0000 [2663857.871177] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2663857.882024] RSP: 0018:ffff9c142f5c3b90 EFLAGS: 00000202 [2663857.888259] RAX: 0000000000000001 RBX: ffffffffffffff10 RCX: 0000000000010000 [2663857.896315] RDX: 0000000000210001 RSI: 0000000000810001 RDI: ffff9c2a4a5112dc [2663857.904374] RBP: ffff9c142f5c3b90 R08: ffff9c159f21b880 R09: ffff9c159f49b880 [2663857.912434] R10: ffff9c1469367500 R11: ffff9c1201b18000 R12: 0000000000810000 [2663857.920498] R13: ffff9c159f41b880 R14: 0000000000010000 R15: 0000000000000000 [2663857.928557] FS: 0000000000000000(0000) GS:ffff9c159f200000(0000) knlGS:0000000000000000 [2663857.937584] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663857.944267] CR2: 000000c001379000 CR3: 000000172e636000 CR4: 00000000007607f0 [2663857.952340] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663857.960415] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663857.968481] PKRU: 00000000 [2663857.972139] Call Trace: [2663857.975539] [] queued_spin_lock_slowpath+0xb/0xf [2663857.982751] [] _raw_spin_lock+0x20/0x30 [2663857.989207] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663857.997043] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663858.004624] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663858.012640] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663858.021091] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663858.028161] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663858.036011] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663858.044529] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663858.052550] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663858.061178] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663858.068924] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663858.077611] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663858.085591] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663858.092776] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663858.101055] [] kthread+0xd1/0xe0 [2663858.106816] [] ? insert_kthread_work+0x40/0x40 [2663858.113767] [] ret_from_fork_nospec_begin+0x7/0x21 [2663858.121055] [] ? insert_kthread_work+0x40/0x40 [2663858.127987] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2663861.585496] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.168@o2ib2 added to recovery queue. Health = 900 [2663861.600094] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 6 previous similar messages [2663871.587167] LustreError: 26090:0:(ldlm_lib.c:3292:target_bulk_io()) @@@ truncated bulk READ 0(1048576) req@ffff9c25ffd24850 x1655452743061568/t0(0) o3->b4a3d0ce-cccd-21fc-49c6-907929c7c068@353@gni5:604/0 lens 608/440 e 0 to 0 dl 1585780709 ref 1 fl Interpret:/0/0 rc 0/0 [2663871.587184] Lustre: f2-OST0014: Bulk IO read error with 58a41b91-ff92-3b6f-295a-6e4cbfd49534 (at 288@gni5), client will retry: rc -110 [2663871.587185] Lustre: Skipped 1 previous similar message [2663871.587413] Lustre: f2-OST0014: Bulk IO write error with 51da61b8-98f1-3acf-ac1b-f1fe668d3a95 (at 1071@gni5), client will retry: rc = -110 [2663871.587414] Lustre: Skipped 17 previous similar messages [2663871.652788] LustreError: 26090:0:(ldlm_lib.c:3292:target_bulk_io()) Skipped 11 previous similar messages [2663872.452061] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.158@o2ib2: 0 seconds [2663872.463138] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 8 previous similar messages [2663872.473268] LNetError: 18738:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.10.33.33@o2ib2 added to recovery queue. Health = 900 [2663876.575029] LustreError: 25910:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(1035836) req@ffff9c1de5aae850 x1661790272342160/t0(0) o4->7d0cf35d-8845-c0e4-4579-fc9a390d5dce@1107@gni4:613/0 lens 608/448 e 0 to 0 dl 1585780718 ref 1 fl Interpret:/0/0 rc 0/0 [2663876.601599] LustreError: 25910:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 13 previous similar messages [2663880.103429] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ll_ost00_033:24598] [2663880.110416] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ll_ost00_123:26911] [2663880.110439] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.110463] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.110464] CPU: 4 PID: 26911 Comm: ll_ost00_123 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.110465] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.110466] task: ffff9c1460079070 ti: ffff9c1445fd8000 task.ti: ffff9c1445fd8000 [2663880.110469] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.110470] RSP: 0018:ffff9c1445fdbb90 EFLAGS: 00000246 [2663880.110471] RAX: 0000000000000000 RBX: ffff9c1445fdbadc RCX: 0000000000210000 [2663880.110471] RDX: ffff9c159f35b880 RSI: 0000000000510001 RDI: ffff9c2a4a5112dc [2663880.110472] RBP: ffff9c1445fdbb90 R08: ffff9c159f29b880 R09: 0000000000000000 [2663880.110472] R10: ffff9c2b0e592100 R11: ffff9c0c5c0d0000 R12: 0000000000000002 [2663880.110473] R13: ffff9c065d579f80 R14: ffff9c1445fdbbd8 R15: 00000000047e5fff [2663880.110474] FS: 0000000000000000(0000) GS:ffff9c159f280000(0000) knlGS:0000000000000000 [2663880.110475] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.110475] CR2: 000000c00130d010 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.110476] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.110476] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.110477] PKRU: 00000000 [2663880.110477] Call Trace: [2663880.110481] [] queued_spin_lock_slowpath+0xb/0xf [2663880.110483] [] _raw_spin_lock+0x20/0x30 [2663880.110522] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.110537] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.110555] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.110576] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.110603] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.110627] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.110650] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.110655] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.110675] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.110695] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.110697] [] ? wake_up_state+0x20/0x20 [2663880.110716] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.110735] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.110737] [] kthread+0xd1/0xe0 [2663880.110739] [] ? insert_kthread_work+0x40/0x40 [2663880.110740] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.110742] [] ? insert_kthread_work+0x40/0x40 [2663880.110756] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.119391] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ll_ost00_131:26919] [2663880.119410] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.119430] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.119431] CPU: 6 PID: 26919 Comm: ll_ost00_131 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.119432] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.119433] task: ffff9c148ed2c1c0 ti: ffff9c1448394000 task.ti: ffff9c1448394000 [2663880.119435] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.119436] RSP: 0018:ffff9c1448397b90 EFLAGS: 00000246 [2663880.119436] RAX: 0000000000000000 RBX: ffff9c1448397adc RCX: 0000000000310000 [2663880.119437] RDX: ffff9c159f31b880 RSI: 0000000000410000 RDI: ffff9c2a4a5112dc [2663880.119437] RBP: ffff9c1448397b90 R08: ffff9c159f2db880 R09: 0000000000000000 [2663880.119438] R10: ffff9c133da57900 R11: ffff9bfefdbb0000 R12: 0000000000000002 [2663880.119438] R13: ffff9c023cf62d00 R14: ffff9c1448397bd8 R15: 00000000047e5fff [2663880.119439] FS: 0000000000000000(0000) GS:ffff9c159f2c0000(0000) knlGS:0000000000000000 [2663880.119440] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.119440] CR2: 00007fcb82c09000 CR3: 000000190da10000 CR4: 00000000007607e0 [2663880.119441] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.119442] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.119442] PKRU: 00000000 [2663880.119442] Call Trace: [2663880.119445] [] queued_spin_lock_slowpath+0xb/0xf [2663880.119447] [] _raw_spin_lock+0x20/0x30 [2663880.119467] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.119482] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.119499] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.119519] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.119544] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.119568] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.119591] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.119595] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.119617] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.119636] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.119639] [] ? __wake_up+0x13/0x20 [2663880.119659] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.119678] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.119680] [] kthread+0xd1/0xe0 [2663880.119681] [] ? insert_kthread_work+0x40/0x40 [2663880.119683] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.119684] [] ? insert_kthread_work+0x40/0x40 [2663880.119698] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.127371] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ll_ost00_112:26900] [2663880.127393] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.127416] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.127418] CPU: 8 PID: 26900 Comm: ll_ost00_112 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.127418] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.127419] task: ffff9c14446341c0 ti: ffff9c1450cc4000 task.ti: ffff9c1450cc4000 [2663880.127422] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.127423] RSP: 0018:ffff9c1450cc7b90 EFLAGS: 00000246 [2663880.127423] RAX: 0000000000000000 RBX: ffff9c1450cc7adc RCX: 0000000000410000 [2663880.127424] RDX: ffff9c159f25b880 RSI: 0000000000110000 RDI: ffff9c2a4a5112dc [2663880.127425] RBP: ffff9c1450cc7b90 R08: ffff9c159f31b880 R09: 0000000000000000 [2663880.127425] R10: ffff9c146b4bc500 R11: ffff9c0dacf38000 R12: 0000000000000002 [2663880.127426] R13: ffff9c1429192640 R14: ffff9c1450cc7bd8 R15: 000000000480afff [2663880.127427] FS: 0000000000000000(0000) GS:ffff9c159f300000(0000) knlGS:0000000000000000 [2663880.127427] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.127428] CR2: 00007f9d0f438000 CR3: 0000002fa937e000 CR4: 00000000007607e0 [2663880.127429] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.127429] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.127430] PKRU: 00000000 [2663880.127430] Call Trace: [2663880.127433] [] queued_spin_lock_slowpath+0xb/0xf [2663880.127435] [] _raw_spin_lock+0x20/0x30 [2663880.127477] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.127491] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.127509] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.127529] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.127555] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.127579] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.127601] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.127607] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.127627] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.127646] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.127648] [] ? queued_spin_lock_slowpath+0xb/0xf [2663880.127649] [] ? _raw_spin_lock+0x20/0x30 [2663880.127668] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.127688] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.127690] [] kthread+0xd1/0xe0 [2663880.127692] [] ? insert_kthread_work+0x40/0x40 [2663880.127694] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.127695] [] ? insert_kthread_work+0x40/0x40 [2663880.127709] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.135352] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ll_ost00_070:24739] [2663880.135371] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.135391] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.135392] CPU: 10 PID: 24739 Comm: ll_ost00_070 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.135393] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.135394] task: ffff9c2c947820e0 ti: ffff9c1434898000 task.ti: ffff9c1434898000 [2663880.135396] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.135397] RSP: 0018:ffff9c143489bb90 EFLAGS: 00000246 [2663880.135397] RAX: 0000000000000000 RBX: ffff9c143489badc RCX: 0000000000510000 [2663880.135398] RDX: ffff9c159f4db880 RSI: 0000000000b10000 RDI: ffff9c2a4a5112dc [2663880.135398] RBP: ffff9c143489bb90 R08: ffff9c159f35b880 R09: 0000000000000000 [2663880.135399] R10: ffff9c123814ad00 R11: ffff9c0b5f498000 R12: 0000000000000002 [2663880.135399] R13: ffff9c0925ed21c0 R14: ffff9c143489bbd8 R15: 00000000047e5fff [2663880.135400] FS: 0000000000000000(0000) GS:ffff9c159f340000(0000) knlGS:0000000000000000 [2663880.135401] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.135402] CR2: 000000c0015ed000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.135402] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.135403] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.135403] PKRU: 00000000 [2663880.135404] Call Trace: [2663880.135406] [] queued_spin_lock_slowpath+0xb/0xf [2663880.135407] [] _raw_spin_lock+0x20/0x30 [2663880.135428] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.135443] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.135460] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.135480] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.135505] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.135529] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.135552] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.135556] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.135576] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.135596] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.135615] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663880.135617] [] ? queued_spin_lock_slowpath+0xb/0xf [2663880.135618] [] ? _raw_spin_lock+0x20/0x30 [2663880.135637] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.135657] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.135658] [] kthread+0xd1/0xe0 [2663880.135660] [] ? insert_kthread_work+0x40/0x40 [2663880.135662] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.135663] [] ? insert_kthread_work+0x40/0x40 [2663880.135677] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.143335] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ll_ost00_073:24742] [2663880.143356] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.143377] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.143379] CPU: 12 PID: 24742 Comm: ll_ost00_073 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.143379] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.143380] task: ffff9c1438eb0000 ti: ffff9c1438830000 task.ti: ffff9c1438830000 [2663880.143383] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.143384] RSP: 0018:ffff9c1438833b90 EFLAGS: 00000246 [2663880.143385] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000610000 [2663880.143385] RDX: ffff9c159f45b880 RSI: 0000000000910000 RDI: ffff9c2a4a5112dc [2663880.143386] RBP: ffff9c1438833b90 R08: ffff9c159f39b880 R09: 0000000000000000 [2663880.143386] R10: ffff9c146184bd00 R11: ffff9c0f017c0000 R12: 0000000000910000 [2663880.143387] R13: ffff9c159f45b880 R14: 0000000000610000 R15: 0000000000000000 [2663880.143388] FS: 0000000000000000(0000) GS:ffff9c159f380000(0000) knlGS:0000000000000000 [2663880.143388] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.143389] CR2: 000000c000f8d000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.143390] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.143390] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.143391] PKRU: 00000000 [2663880.143391] Call Trace: [2663880.143394] [] queued_spin_lock_slowpath+0xb/0xf [2663880.143396] [] _raw_spin_lock+0x20/0x30 [2663880.143435] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.143450] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.143468] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.143488] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.143514] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.143538] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.143560] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.143565] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.143585] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.143604] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.143606] [] ? queued_spin_lock_slowpath+0xb/0xf [2663880.143607] [] ? _raw_spin_lock+0x20/0x30 [2663880.143627] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.143646] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.143648] [] kthread+0xd1/0xe0 [2663880.143650] [] ? insert_kthread_work+0x40/0x40 [2663880.143651] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.143653] [] ? insert_kthread_work+0x40/0x40 [2663880.143667] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.151312] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [ll_ost00_089:24777] [2663880.151332] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.151352] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.151353] CPU: 14 PID: 24777 Comm: ll_ost00_089 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.151354] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.151355] task: ffff9c143c438000 ti: ffff9c143beb4000 task.ti: ffff9c143beb4000 [2663880.151357] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.151358] RSP: 0018:ffff9c143beb7b90 EFLAGS: 00000246 [2663880.151358] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000710000 [2663880.151359] RDX: ffff9c159f39b880 RSI: 0000000000610000 RDI: ffff9c2a4a5112dc [2663880.151359] RBP: ffff9c143beb7b90 R08: ffff9c159f3db880 R09: 0000000000000000 [2663880.151360] R10: ffff9c12cda40d00 R11: ffff9c07b8908000 R12: 0000000000610000 [2663880.151360] R13: ffff9c159f39b880 R14: 0000000000710000 R15: 0000000000000000 [2663880.151361] FS: 0000000000000000(0000) GS:ffff9c159f3c0000(0000) knlGS:0000000000000000 [2663880.151362] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.151363] CR2: 00007fcb82c09000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.151363] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.151364] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.151364] PKRU: 00000000 [2663880.151365] Call Trace: [2663880.151367] [] queued_spin_lock_slowpath+0xb/0xf [2663880.151368] [] _raw_spin_lock+0x20/0x30 [2663880.151390] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.151404] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.151422] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.151441] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.151467] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.151491] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.151513] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.151517] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.151538] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.151557] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.151576] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663880.151578] [] ? queued_spin_lock_slowpath+0xb/0xf [2663880.151579] [] ? _raw_spin_lock+0x20/0x30 [2663880.151599] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.151618] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.151620] [] kthread+0xd1/0xe0 [2663880.151621] [] ? insert_kthread_work+0x40/0x40 [2663880.151623] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.151624] [] ? insert_kthread_work+0x40/0x40 [2663880.151638] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.160292] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ll_ost00_092:24784] [2663880.160316] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.160341] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.160343] CPU: 16 PID: 24784 Comm: ll_ost00_092 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.160344] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.160344] task: ffff9c143d550000 ti: ffff9c1455194000 task.ti: ffff9c1455194000 [2663880.160386] RIP: 0010:[] [] ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663880.160387] RSP: 0018:ffff9c1455197a38 EFLAGS: 00000206 [2663880.160387] RAX: ffff9c0e661957c0 RBX: ffff9bfedf1c7580 RCX: 000000000000001a [2663880.160388] RDX: 0000000000000004 RSI: 000000000480a000 RDI: ffff9c14503ef000 [2663880.160388] RBP: ffff9c1455197ac0 R08: ffff9c2a4a5112f0 R09: 000000000480afff [2663880.160389] R10: 000000000480a000 R11: 000000000480a000 R12: ffff9c1455197b70 [2663880.160389] R13: ffff9bfedf1c7580 R14: 0000000000000010 R15: ffffffff94223abd [2663880.160390] FS: 0000000000000000(0000) GS:ffff9c159f400000(0000) knlGS:0000000000000000 [2663880.160391] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.160392] CR2: 000000c000eff000 CR3: 0000001724304000 CR4: 00000000007607e0 [2663880.160392] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.160393] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.160393] PKRU: 00000000 [2663880.160394] Call Trace: [2663880.160411] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663880.160427] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663880.160442] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663880.160456] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663880.160471] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.160487] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.160508] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.160534] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.160558] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.160580] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.160586] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.160606] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.160626] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.160645] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.160665] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.160667] [] kthread+0xd1/0xe0 [2663880.160669] [] ? insert_kthread_work+0x40/0x40 [2663880.160671] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.160672] [] ? insert_kthread_work+0x40/0x40 [2663880.160686] Code: b8 00 00 00 74 2f 48 8b 50 60 49 39 d0 48 8d 42 a0 0f 84 c7 00 00 00 48 39 f3 0f 84 de 02 00 00 49 39 c7 74 e1 8b 90 98 00 00 00 <44> 85 24 95 00 01 5a c1 75 c1 83 c1 01 48 89 f7 48 8b 90 d0 00 [2663880.168271] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ll_ost00_038:24607] [2663880.168291] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.168311] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.168313] CPU: 18 PID: 24607 Comm: ll_ost00_038 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.168314] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.168314] task: ffff9c1551e720e0 ti: ffff9c143e1ac000 task.ti: ffff9c143e1ac000 [2663880.168317] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.168317] RSP: 0018:ffff9c143e1afb90 EFLAGS: 00000246 [2663880.168318] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000910000 [2663880.168319] RDX: ffff9c2d9d8db880 RSI: 0000000000a10001 RDI: ffff9c2a4a5112dc [2663880.168319] RBP: ffff9c143e1afb90 R08: ffff9c159f45b880 R09: 0000000000000000 [2663880.168320] R10: ffff9c1f9b4d0500 R11: ffff9c00f5008000 R12: 0000000000a10000 [2663880.168320] R13: ffff9c159f49b880 R14: 0000000000910000 R15: 0000000000000000 [2663880.168321] FS: 0000000000000000(0000) GS:ffff9c159f440000(0000) knlGS:0000000000000000 [2663880.168322] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.168323] CR2: 000000c000df3010 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.168323] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.168324] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.168324] PKRU: 00000000 [2663880.168325] Call Trace: [2663880.168328] [] queued_spin_lock_slowpath+0xb/0xf [2663880.168330] [] _raw_spin_lock+0x20/0x30 [2663880.168368] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.168382] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.168400] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.168419] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.168445] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.168469] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.168492] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.168497] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.168517] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.168537] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.168539] [] ? __wake_up+0x13/0x20 [2663880.168558] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.168578] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.168580] [] kthread+0xd1/0xe0 [2663880.168581] [] ? insert_kthread_work+0x40/0x40 [2663880.168583] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.168584] [] ? insert_kthread_work+0x40/0x40 [2663880.168598] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.176252] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_ost00_049:24692] [2663880.176271] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.176291] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.176293] CPU: 20 PID: 24692 Comm: ll_ost00_049 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.176293] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.176294] task: ffff9c152a34b150 ti: ffff9c1443e64000 task.ti: ffff9c1443e64000 [2663880.176296] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.176297] RSP: 0018:ffff9c1443e67b90 EFLAGS: 00000246 [2663880.176298] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000a10000 [2663880.176298] RDX: ffff9c159f21b880 RSI: 0000000000010000 RDI: ffff9c2a4a5112dc [2663880.176299] RBP: ffff9c1443e67b90 R08: ffff9c159f49b880 R09: 0000000000000000 [2663880.176299] R10: ffff9c146a96e500 R11: ffff9c0785538000 R12: 0000000000b90000 [2663880.176300] R13: ffff9c2d9d8db880 R14: 0000000000a10000 R15: 0000000000000000 [2663880.176301] FS: 0000000000000000(0000) GS:ffff9c159f480000(0000) knlGS:0000000000000000 [2663880.176302] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.176302] CR2: 00007f77d9493000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.176303] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.176303] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.176304] PKRU: 00000000 [2663880.176304] Call Trace: [2663880.176306] [] queued_spin_lock_slowpath+0xb/0xf [2663880.176308] [] _raw_spin_lock+0x20/0x30 [2663880.176332] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.176347] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.176364] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.176384] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.176409] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.176434] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.176456] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.176461] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.176482] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.176501] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.176521] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663880.176523] [] ? queued_spin_lock_slowpath+0xb/0xf [2663880.176524] [] ? _raw_spin_lock+0x20/0x30 [2663880.176543] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.176563] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.176564] [] kthread+0xd1/0xe0 [2663880.176566] [] ? insert_kthread_work+0x40/0x40 [2663880.176568] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.176569] [] ? insert_kthread_work+0x40/0x40 [2663880.176583] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.184233] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ll_ost00_138:26926] [2663880.184252] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.184273] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.184274] CPU: 22 PID: 26926 Comm: ll_ost00_138 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.184275] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.184276] task: ffff9c1432feb150 ti: ffff9c1448310000 task.ti: ffff9c1448310000 [2663880.184279] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.184279] RSP: 0018:ffff9c1448313b90 EFLAGS: 00000246 [2663880.184280] RAX: 0000000000000000 RBX: ffff9c1448313adc RCX: 0000000000b10000 [2663880.184280] RDX: ffff9c159f2db880 RSI: 0000000000310000 RDI: ffff9c2a4a5112dc [2663880.184281] RBP: ffff9c1448313b90 R08: ffff9c159f4db880 R09: 0000000000000000 [2663880.184281] R10: ffff9c272c285900 R11: ffff9c06f6418000 R12: 0000000000000002 [2663880.184282] R13: ffff9c0ce138ad00 R14: ffff9c1448313bd8 R15: 000000000480efff [2663880.184283] FS: 0000000000000000(0000) GS:ffff9c159f4c0000(0000) knlGS:0000000000000000 [2663880.184284] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.184284] CR2: 000000c0013a7000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663880.184285] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.184286] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.184286] PKRU: 00000000 [2663880.184286] Call Trace: [2663880.184289] [] queued_spin_lock_slowpath+0xb/0xf [2663880.184291] [] _raw_spin_lock+0x20/0x30 [2663880.184325] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.184340] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.184358] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663880.184378] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663880.184404] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663880.184427] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663880.184450] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663880.184455] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663880.184475] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.184494] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.184496] [] ? queued_spin_lock_slowpath+0xb/0xf [2663880.184497] [] ? _raw_spin_lock+0x20/0x30 [2663880.184517] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.184537] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.184539] [] kthread+0xd1/0xe0 [2663880.184540] [] ? insert_kthread_work+0x40/0x40 [2663880.184542] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.184543] [] ? insert_kthread_work+0x40/0x40 [2663880.184557] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663880.188224] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ldlm_cn01_046:57960] [2663880.188245] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663880.188266] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663880.188268] CPU: 23 PID: 57960 Comm: ldlm_cn01_046 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663880.188268] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663880.188269] task: ffff9c152e665230 ti: ffff9c0f7f358000 task.ti: ffff9c0f7f358000 [2663880.188272] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663880.188273] RSP: 0018:ffff9c0f7f35bc18 EFLAGS: 00000246 [2663880.188274] RAX: 0000000000000000 RBX: ffff9c0f7f35bbc0 RCX: 0000000000b90000 [2663880.188274] RDX: ffff9c159f49b880 RSI: 0000000000a10000 RDI: ffff9c2a4a5112dc [2663880.188275] RBP: ffff9c0f7f35bc18 R08: ffff9c2d9d8db880 R09: 0000000000000000 [2663880.188275] R10: 0000000000000001 R11: ffff9c2a27221400 R12: ffff9c2a27221400 [2663880.188276] R13: 0000000000000292 R14: 0000000000000000 R15: ffff9c2a27221400 [2663880.188277] FS: 0000000000000000(0000) GS:ffff9c2d9d8c0000(0000) knlGS:0000000000000000 [2663880.188278] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663880.188278] CR2: 00007f39ae98dbc0 CR3: 000000190da10000 CR4: 00000000007607e0 [2663880.188279] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663880.188279] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663880.188280] PKRU: 00000000 [2663880.188280] Call Trace: [2663880.188284] [] queued_spin_lock_slowpath+0xb/0xf [2663880.188285] [] _raw_spin_lock+0x20/0x30 [2663880.188317] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663880.188331] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663880.188348] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663880.188365] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663880.188381] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663880.188401] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663880.188421] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663880.188423] [] ? __wake_up+0x13/0x20 [2663880.188442] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663880.188462] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663880.188464] [] kthread+0xd1/0xe0 [2663880.188465] [] ? insert_kthread_work+0x40/0x40 [2663880.188467] [] ret_from_fork_nospec_begin+0x7/0x21 [2663880.188468] [] ? insert_kthread_work+0x40/0x40 [2663880.188482] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663881.761409] Lustre: 23737:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585780623/real 1585780623] req@ffff9c24b0d67980 x1660035340779008/t0(0) o106->f2-OST0038@102@gni4:15/16 lens 296/280 e 0 to 1 dl 1585780679 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [2663884.089732] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ll_ost00_056:24709] [2663884.089757] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663884.089780] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663884.089782] CPU: 0 PID: 24709 Comm: ll_ost00_056 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663884.089783] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663884.089784] task: ffff9c1595ed62a0 ti: ffff9c142f5c0000 task.ti: ffff9c142f5c0000 [2663884.089787] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x156/0x200 [2663884.089788] RSP: 0018:ffff9c142f5c3b90 EFLAGS: 00000202 [2663884.089788] RAX: 0000000000000001 RBX: ffffffffffffff10 RCX: 0000000000010000 [2663884.089789] RDX: 0000000000210001 RSI: 0000000000810001 RDI: ffff9c2a4a5112dc [2663884.089789] RBP: ffff9c142f5c3b90 R08: ffff9c159f21b880 R09: ffff9c159f49b880 [2663884.089790] R10: ffff9c1469367500 R11: ffff9c1201b18000 R12: 0000000000810000 [2663884.089790] R13: ffff9c159f41b880 R14: 0000000000010000 R15: 0000000000000000 [2663884.089791] FS: 0000000000000000(0000) GS:ffff9c159f200000(0000) knlGS:0000000000000000 [2663884.089792] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663884.089793] CR2: 000000c001379000 CR3: 000000172e636000 CR4: 00000000007607f0 [2663884.089793] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663884.089794] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663884.089794] PKRU: 00000000 [2663884.089794] Call Trace: [2663884.089798] [] queued_spin_lock_slowpath+0xb/0xf [2663884.089800] [] _raw_spin_lock+0x20/0x30 [2663884.089827] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.089841] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.089859] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.089879] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.089905] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.089929] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.089952] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.089956] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.089976] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.089996] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.090015] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.090034] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.090053] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.090072] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.090074] [] kthread+0xd1/0xe0 [2663884.090075] [] ? insert_kthread_work+0x40/0x40 [2663884.090077] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.090079] [] ? insert_kthread_work+0x40/0x40 [2663884.090093] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [2663884.422934] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.158@o2ib2: 1 seconds [2663884.422939] LNetError: 18738:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.10.33.33@o2ib2 added to recovery queue. Health = 900 [2663884.975579] INFO: rcu_sched self-detected stall on CPU { 14} [2663884.975588] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.975589] INFO: rcu_sched self-detected stall on CPU { 18} (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.975589] Task dump for CPU 0: [2663884.975591] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663884.975592] Call Trace: [2663884.975625] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.975628] [] ? ktime_get_ts64+0x52/0xf0 [2663884.975634] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.975650] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.975679] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.975681] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663884.975684] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.975686] [] ? _raw_spin_lock+0x20/0x30 [2663884.975702] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.975717] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.975734] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.975754] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.975781] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.975805] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.975828] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.975832] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.975854] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.975875] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.975894] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.975913] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.975932] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.975951] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.975953] [] ? kthread+0xd1/0xe0 [2663884.975954] [] ? insert_kthread_work+0x40/0x40 [2663884.975956] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.975957] [] ? insert_kthread_work+0x40/0x40 [2663884.975958] Task dump for CPU 2: [2663884.975960] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663884.975960] Call Trace: [2663884.975976] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.975978] [] ? ktime_get_ts64+0x52/0xf0 [2663884.975982] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.975997] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.976009] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.976011] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.976013] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.976014] [] ? _raw_spin_lock+0x20/0x30 [2663884.976028] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.976042] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.976059] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.976078] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.976102] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.976125] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.976147] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.976150] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.976171] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.976190] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.976208] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.976227] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.976245] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.976264] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.976265] [] ? kthread+0xd1/0xe0 [2663884.976267] [] ? insert_kthread_work+0x40/0x40 [2663884.976268] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.976270] [] ? insert_kthread_work+0x40/0x40 [2663884.976270] Task dump for CPU 4: [2663884.976272] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663884.976272] Call Trace: [2663884.976295] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.976318] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.976339] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.976343] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.976362] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.976381] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.976382] [] ? wake_up_state+0x20/0x20 [2663884.976401] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.976420] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.976422] [] kthread+0xd1/0xe0 [2663884.976423] [] ? insert_kthread_work+0x40/0x40 [2663884.976425] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.976426] [] ? insert_kthread_work+0x40/0x40 [2663884.976427] Task dump for CPU 6: [2663884.976428] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663884.976429] Call Trace: [2663884.976444] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.976446] [] ? ktime_get_ts64+0x52/0xf0 [2663884.976450] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.976466] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.976477] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.976479] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.976481] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.976482] [] ? _raw_spin_lock+0x20/0x30 [2663884.976497] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.976511] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.976528] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.976547] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.976571] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.976596] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.976598] INFO: rcu_sched self-detected stall on CPU [2663884.976599] INFO: rcu_sched self-detected stall on CPU [2663884.976600] INFO: rcu_sched self-detected stall on CPU [2663884.976601] INFO: rcu_sched self-detected stall on CPU [2663884.976602] INFO: rcu_sched self-detected stall on CPU [2663884.976602] INFO: rcu_sched self-detected stall on CPU [2663884.976603] INFO: rcu_sched self-detected stall on CPU [2663884.976604] INFO: rcu_sched self-detected stall on CPU [2663884.976604] INFO: rcu_sched self-detected stall on CPU [2663884.976626] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.976626] INFO: rcu_sched self-detected stall on CPU [2663884.976627] { [2663884.976628] { [2663884.976629] { [2663884.976629] { [2663884.976629] { [2663884.976630] { [2663884.976630] { [2663884.976631] { [2663884.976631] { [2663884.976635] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.976635] { [2663884.976636] 12 [2663884.976637] 23 [2663884.976638] 16 [2663884.976639] 10 [2663884.976639] 20 [2663884.976640] 6 [2663884.976640] 0 [2663884.976641] 8 [2663884.976641] 22 [2663884.976661] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.976661] 4 [2663884.976662] } [2663884.976663] } [2663884.976663] } [2663884.976663] } [2663884.976664] } [2663884.976664] } [2663884.976665] } [2663884.976665] } [2663884.976665] } [2663884.976685] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.976686] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976687] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976688] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976688] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976690] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976691] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976692] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976693] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976694] (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976695] } (t=60000 jiffies g=209325237 c=209325236 q=4700) [2663884.976698] [] ? __wake_up+0x13/0x20 [2663884.976716] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.976735] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.976737] [] ? kthread+0xd1/0xe0 [2663884.976738] [] ? insert_kthread_work+0x40/0x40 [2663884.976740] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.976741] [] ? insert_kthread_work+0x40/0x40 [2663884.976742] Task dump for CPU 8: [2663884.976743] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663884.976743] Call Trace: [2663884.976759] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.976774] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663884.976790] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.976801] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.976802] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.976804] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.976805] [] ? _raw_spin_lock+0x20/0x30 [2663884.976819] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.976833] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.976850] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.976869] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.976892] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.976915] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.976937] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.976941] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.976961] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.976980] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.976981] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.976982] [] ? _raw_spin_lock+0x20/0x30 [2663884.977001] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.977020] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.977021] [] ? kthread+0xd1/0xe0 [2663884.977023] [] ? insert_kthread_work+0x40/0x40 [2663884.977024] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.977025] [] ? insert_kthread_work+0x40/0x40 [2663884.977026] Task dump for CPU 10: [2663884.977027] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663884.977028] Call Trace: [2663884.977043] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.977045] [] ? ktime_get_ts64+0x52/0xf0 [2663884.977048] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.977064] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.977075] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.977077] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.977078] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.977079] [] ? _raw_spin_lock+0x20/0x30 [2663884.977093] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.977108] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.977124] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.977143] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.977166] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.977189] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.977211] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.977214] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.977234] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.977253] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.977272] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.977273] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.977274] [] ? _raw_spin_lock+0x20/0x30 [2663884.977293] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.977312] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.977313] [] ? kthread+0xd1/0xe0 [2663884.977315] [] ? insert_kthread_work+0x40/0x40 [2663884.977316] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.977318] [] ? insert_kthread_work+0x40/0x40 [2663884.977318] Task dump for CPU 12: [2663884.977319] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663884.977320] Call Trace: [2663884.977335] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.977337] [] ? ktime_get_ts64+0x52/0xf0 [2663884.977341] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.977356] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.977367] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.977369] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.977370] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.977371] [] ? _raw_spin_lock+0x20/0x30 [2663884.977386] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.977400] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.977416] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.977435] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.977459] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.977482] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.977504] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.977508] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.977527] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.977546] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.977547] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.977548] [] ? _raw_spin_lock+0x20/0x30 [2663884.977568] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.977587] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.977588] [] ? kthread+0xd1/0xe0 [2663884.977589] [] ? insert_kthread_work+0x40/0x40 [2663884.977591] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.977592] [] ? insert_kthread_work+0x40/0x40 [2663884.977593] Task dump for CPU 14: [2663884.977594] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663884.977594] Call Trace: [2663884.977596] [] sched_show_task+0xa8/0x110 [2663884.977598] [] dump_cpu_task+0x39/0x70 [2663884.977601] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663884.977603] [] rcu_check_callbacks+0x442/0x730 [2663884.977605] [] ? tick_sched_do_timer+0x50/0x50 [2663884.977607] [] update_process_times+0x46/0x80 [2663884.977609] [] tick_sched_handle+0x30/0x70 [2663884.977610] [] tick_sched_timer+0x39/0x80 [2663884.977612] [] __hrtimer_run_queues+0x10e/0x270 [2663884.977614] [] hrtimer_interrupt+0xaf/0x1d0 [2663884.977617] [] local_apic_timer_interrupt+0x3b/0x60 [2663884.977619] [] smp_apic_timer_interrupt+0x43/0x60 [2663884.977621] [] apic_timer_interrupt+0x16a/0x170 [2663884.977623] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.977624] [] queued_spin_lock_slowpath+0xb/0xf [2663884.977625] [] _raw_spin_lock+0x20/0x30 [2663884.977640] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.977654] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.977671] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.977690] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.977714] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.977737] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.977759] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.977763] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.977782] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.977801] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.977819] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.977821] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.977822] [] ? _raw_spin_lock+0x20/0x30 [2663884.977841] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.977859] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.977861] [] kthread+0xd1/0xe0 [2663884.977862] [] ? insert_kthread_work+0x40/0x40 [2663884.977864] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.977865] [] ? insert_kthread_work+0x40/0x40 [2663884.977866] Task dump for CPU 16: [2663884.977866] Task dump for CPU 0: [2663884.977868] ll_ost00_092 R [2663884.977868] ll_ost00_056 R [2663884.977868] running task [2663884.977869] 0 24784 2 0x00000008 [2663884.977870] running task 0 24709 2 0x00000008 [2663884.977870] Call Trace: [2663884.977871] Call Trace: [2663884.977886] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663884.977906] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.977922] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663884.977924] [] ? ktime_get_ts64+0x52/0xf0 [2663884.977939] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663884.977943] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.977957] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663884.977973] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.977987] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663884.978002] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.978016] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663884.978018] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663884.978032] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.978034] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.978050] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.978051] [] ? _raw_spin_lock+0x20/0x30 [2663884.978070] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.978084] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.978108] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.978122] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.978145] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.978162] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.978183] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.978203] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.978207] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.978231] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.978250] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.978274] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.978293] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.978315] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.978334] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.978337] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.978356] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.978375] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.978376] [] ? kthread+0xd1/0xe0 [2663884.978396] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.978397] [] ? insert_kthread_work+0x40/0x40 [2663884.978416] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.978417] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.978436] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.978437] [] ? insert_kthread_work+0x40/0x40 [2663884.978456] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.978456] Task dump for CPU 18: [2663884.978475] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.978477] [] ? kthread+0xd1/0xe0 [2663884.978478] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663884.978479] [] ? insert_kthread_work+0x40/0x40 [2663884.978479] Call Trace: [2663884.978481] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.978496] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.978497] [] ? insert_kthread_work+0x40/0x40 [2663884.978499] [] ? ktime_get_ts64+0x52/0xf0 [2663884.978499] Task dump for CPU 2: [2663884.978503] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.978518] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.978530] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.978531] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663884.978532] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.978533] Call Trace: [2663884.978534] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.978550] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.978551] [] ? _raw_spin_lock+0x20/0x30 [2663884.978553] [] ? ktime_get_ts64+0x52/0xf0 [2663884.978567] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.978570] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.978585] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.978600] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.978616] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.978628] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.978647] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.978648] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.978671] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.978673] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.978696] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.978698] [] ? _raw_spin_lock+0x20/0x30 [2663884.978719] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.978733] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.978737] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.978751] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.978770] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.978786] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.978805] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.978824] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.978826] [] ? __wake_up+0x13/0x20 [2663884.978849] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.978868] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.978891] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.978910] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.978932] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.978933] [] ? kthread+0xd1/0xe0 [2663884.978936] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.978938] [] ? insert_kthread_work+0x40/0x40 [2663884.978957] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.978958] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.978977] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.978979] [] ? insert_kthread_work+0x40/0x40 [2663884.978997] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.978998] Task dump for CPU 20: [2663884.979016] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.979036] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.979037] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663884.979055] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.979056] Call Trace: [2663884.979057] [] ? kthread+0xd1/0xe0 [2663884.979072] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.979074] [] ? insert_kthread_work+0x40/0x40 [2663884.979075] [] ? ktime_get_ts64+0x52/0xf0 [2663884.979077] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.979080] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.979081] [] ? insert_kthread_work+0x40/0x40 [2663884.979097] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.979097] Task dump for CPU 4: [2663884.979109] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.979110] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663884.979111] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.979112] Call Trace: [2663884.979113] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.979136] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.979137] [] ? _raw_spin_lock+0x20/0x30 [2663884.979160] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.979174] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.979195] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.979209] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.979213] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.979229] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.979248] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.979267] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.979286] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.979309] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.979311] [] ? wake_up_state+0x20/0x20 [2663884.979334] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.979353] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.979375] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.979393] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.979397] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.979398] [] kthread+0xd1/0xe0 [2663884.979418] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.979419] [] ? insert_kthread_work+0x40/0x40 [2663884.979438] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.979439] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.979458] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.979459] [] ? insert_kthread_work+0x40/0x40 [2663884.979461] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.979461] Task dump for CPU 6: [2663884.979462] [] ? _raw_spin_lock+0x20/0x30 [2663884.979481] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.979483] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663884.979501] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.979501] Call Trace: [2663884.979503] [] ? kthread+0xd1/0xe0 [2663884.979518] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.979520] [] ? insert_kthread_work+0x40/0x40 [2663884.979521] [] ? ktime_get_ts64+0x52/0xf0 [2663884.979523] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.979526] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.979527] [] ? insert_kthread_work+0x40/0x40 [2663884.979542] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.979543] Task dump for CPU 22: [2663884.979554] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.979556] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.979557] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663884.979559] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.979559] Call Trace: [2663884.979560] [] ? _raw_spin_lock+0x20/0x30 [2663884.979576] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.979590] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.979592] [] ? ktime_get_ts64+0x52/0xf0 [2663884.979605] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.979609] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.979625] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.979640] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.979659] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.979671] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.979694] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.979696] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.979719] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.979720] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.979742] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.979743] [] ? _raw_spin_lock+0x20/0x30 [2663884.979746] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.979761] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.979779] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.979793] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.979812] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.979828] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.979830] [] ? __wake_up+0x13/0x20 [2663884.979849] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.979867] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.979890] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.979908] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.979931] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.979933] [] ? kthread+0xd1/0xe0 [2663884.979955] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.979956] [] ? insert_kthread_work+0x40/0x40 [2663884.979959] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.979961] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.979980] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.979981] [] ? insert_kthread_work+0x40/0x40 [2663884.980000] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.980001] Task dump for CPU 8: [2663884.980002] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980004] [] ? _raw_spin_lock+0x20/0x30 [2663884.980005] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663884.980024] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.980024] Call Trace: [2663884.980043] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.980058] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.980060] [] ? kthread+0xd1/0xe0 [2663884.980075] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663884.980076] [] ? insert_kthread_work+0x40/0x40 [2663884.980091] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.980092] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.980104] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.980105] [] ? insert_kthread_work+0x40/0x40 [2663884.980106] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.980107] Task dump for CPU 23: [2663884.980109] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980111] [] ? _raw_spin_lock+0x20/0x30 [2663884.980125] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.980126] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663884.980140] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.980140] Call Trace: [2663884.980156] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.980173] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663884.980192] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.980208] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663884.980232] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.980251] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.980274] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.980293] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.980315] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.980316] [] ? __wake_up+0x13/0x20 [2663884.980320] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.980338] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.980357] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.980375] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.980394] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.980395] [] ? kthread+0xd1/0xe0 [2663884.980397] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980398] [] ? insert_kthread_work+0x40/0x40 [2663884.980400] [] ? _raw_spin_lock+0x20/0x30 [2663884.980401] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.980419] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.980421] [] ? insert_kthread_work+0x40/0x40 [2663884.980439] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.980441] [] ? kthread+0xd1/0xe0 [2663884.980442] [] ? insert_kthread_work+0x40/0x40 [2663884.980443] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.980445] [] ? insert_kthread_work+0x40/0x40 [2663884.980445] Task dump for CPU 10: [2663884.980447] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663884.980447] Call Trace: [2663884.980462] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.980464] [] ? ktime_get_ts64+0x52/0xf0 [2663884.980467] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.980483] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.980494] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.980496] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.980497] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980499] [] ? _raw_spin_lock+0x20/0x30 [2663884.980513] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.980527] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.980544] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.980562] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.980586] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.980610] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.980632] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.980636] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.980655] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.980674] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.980693] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.980694] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980696] [] ? _raw_spin_lock+0x20/0x30 [2663884.980714] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.980733] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.980734] [] ? kthread+0xd1/0xe0 [2663884.980736] [] ? insert_kthread_work+0x40/0x40 [2663884.980737] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.980738] [] ? insert_kthread_work+0x40/0x40 [2663884.980739] Task dump for CPU 12: [2663884.980740] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663884.980740] Call Trace: [2663884.980756] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.980758] [] ? ktime_get_ts64+0x52/0xf0 [2663884.980761] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.980776] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.980788] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.980789] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.980791] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980792] [] ? _raw_spin_lock+0x20/0x30 [2663884.980806] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.980821] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.980837] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.980856] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.980880] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.980903] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.980925] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.980928] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.980948] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.980967] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.980968] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.980969] [] ? _raw_spin_lock+0x20/0x30 [2663884.980988] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.981008] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.981009] [] ? kthread+0xd1/0xe0 [2663884.981010] [] ? insert_kthread_work+0x40/0x40 [2663884.981012] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.981013] [] ? insert_kthread_work+0x40/0x40 [2663884.981014] Task dump for CPU 14: [2663884.981015] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663884.981015] Call Trace: [2663884.981031] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.981032] [] ? ktime_get_ts64+0x52/0xf0 [2663884.981036] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.981051] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.981063] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.981064] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.981066] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.981067] [] ? _raw_spin_lock+0x20/0x30 [2663884.981081] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.981095] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.981112] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.981130] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.981154] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.981177] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.981199] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.981202] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.981222] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.981241] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.981260] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.981261] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.981262] [] ? _raw_spin_lock+0x20/0x30 [2663884.981281] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.981300] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.981301] [] ? kthread+0xd1/0xe0 [2663884.981303] [] ? insert_kthread_work+0x40/0x40 [2663884.981304] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.981305] [] ? insert_kthread_work+0x40/0x40 [2663884.981306] Task dump for CPU 16: [2663884.981307] Task dump for CPU 0: [2663884.981308] ll_ost00_092 R [2663884.981308] ll_ost00_056 R [2663884.981309] running task [2663884.981309] 0 24784 2 0x00000008 [2663884.981310] running task 0 24709 2 0x00000008 [2663884.981311] Call Trace: [2663884.981311] Call Trace: [2663884.981327] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663884.981344] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.981359] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663884.981361] [] ? ktime_get_ts64+0x52/0xf0 [2663884.981376] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663884.981379] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.981394] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663884.981410] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.981424] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663884.981437] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.981451] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663884.981452] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663884.981466] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.981468] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.981484] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.981486] [] ? _raw_spin_lock+0x20/0x30 [2663884.981504] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.981518] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.981541] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.981555] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.981578] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.981595] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.981616] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.981635] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.981639] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.981663] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.981682] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.981706] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.981724] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.981746] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.981765] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.981768] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.981786] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.981806] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.981808] [] ? kthread+0xd1/0xe0 [2663884.981827] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.981828] [] ? insert_kthread_work+0x40/0x40 [2663884.981847] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.981848] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.981867] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.981868] [] ? insert_kthread_work+0x40/0x40 [2663884.981887] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.981887] Task dump for CPU 18: [2663884.981906] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.981908] [] ? kthread+0xd1/0xe0 [2663884.981908] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663884.981910] [] ? insert_kthread_work+0x40/0x40 [2663884.981910] Call Trace: [2663884.981911] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.981913] [] ? insert_kthread_work+0x40/0x40 [2663884.981915] [] sched_show_task+0xa8/0x110 [2663884.981915] Task dump for CPU 2: [2663884.981917] [] dump_cpu_task+0x39/0x70 [2663884.981919] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663884.981921] [] rcu_check_callbacks+0x442/0x730 [2663884.981922] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663884.981923] [] ? tick_sched_do_timer+0x50/0x50 [2663884.981924] Call Trace: [2663884.981926] [] update_process_times+0x46/0x80 [2663884.981941] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.981943] [] tick_sched_handle+0x30/0x70 [2663884.981945] [] ? ktime_get_ts64+0x52/0xf0 [2663884.981946] [] tick_sched_timer+0x39/0x80 [2663884.981950] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.981951] [] __hrtimer_run_queues+0x10e/0x270 [2663884.981967] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.981968] [] hrtimer_interrupt+0xaf/0x1d0 [2663884.981980] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.981982] [] local_apic_timer_interrupt+0x3b/0x60 [2663884.981983] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.981985] [] smp_apic_timer_interrupt+0x43/0x60 [2663884.981987] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.981988] [] apic_timer_interrupt+0x16a/0x170 [2663884.981990] [] ? _raw_spin_lock+0x20/0x30 [2663884.982004] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.982005] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.982019] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.982021] [] queued_spin_lock_slowpath+0xb/0xf [2663884.982037] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.982039] [] _raw_spin_lock+0x20/0x30 [2663884.982058] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.982072] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.982095] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.982109] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.982133] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.982149] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.982171] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.982190] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.982193] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.982216] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.982236] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.982259] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.982278] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.982299] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.982318] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.982321] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.982339] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.982358] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.982376] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.982395] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.982413] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.982415] [] ? __wake_up+0x13/0x20 [2663884.982416] [] ? kthread+0xd1/0xe0 [2663884.982434] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.982436] [] ? insert_kthread_work+0x40/0x40 [2663884.982454] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.982455] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.982457] [] kthread+0xd1/0xe0 [2663884.982458] [] ? insert_kthread_work+0x40/0x40 [2663884.982459] [] ? insert_kthread_work+0x40/0x40 [2663884.982460] Task dump for CPU 4: [2663884.982461] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.982463] [] ? insert_kthread_work+0x40/0x40 [2663884.982464] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663884.982464] Task dump for CPU 20: [2663884.982465] Call Trace: [2663884.982489] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.982490] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663884.982513] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.982514] Call Trace: [2663884.982536] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.982551] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.982555] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.982556] [] ? ktime_get_ts64+0x52/0xf0 [2663884.982575] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.982578] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.982597] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.982612] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.982614] [] ? wake_up_state+0x20/0x20 [2663884.982625] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.982644] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.982645] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.982663] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.982665] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.982666] [] kthread+0xd1/0xe0 [2663884.982667] [] ? _raw_spin_lock+0x20/0x30 [2663884.982669] [] ? insert_kthread_work+0x40/0x40 [2663884.982683] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.982684] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.982698] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.982700] [] ? insert_kthread_work+0x40/0x40 [2663884.982716] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.982717] Task dump for CPU 6: [2663884.982736] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.982760] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.982761] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663884.982784] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.982784] Call Trace: [2663884.982806] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.982822] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.982826] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.982827] [] ? ktime_get_ts64+0x52/0xf0 [2663884.982846] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.982850] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.982869] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.982884] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.982902] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.982913] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.982915] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.982916] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.982917] [] ? _raw_spin_lock+0x20/0x30 [2663884.982919] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.982937] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.982938] [] ? _raw_spin_lock+0x20/0x30 [2663884.982957] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.982971] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.982972] [] ? kthread+0xd1/0xe0 [2663884.982986] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.982987] [] ? insert_kthread_work+0x40/0x40 [2663884.983004] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.983005] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.983024] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.983025] [] ? insert_kthread_work+0x40/0x40 [2663884.983049] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.983049] Task dump for CPU 22: [2663884.983073] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.983095] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.983096] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663884.983099] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.983099] Call Trace: [2663884.983119] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.983134] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663884.983153] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.983155] [] ? ktime_get_ts64+0x52/0xf0 [2663884.983156] [] ? __wake_up+0x13/0x20 [2663884.983160] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663884.983179] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.983194] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663884.983213] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.983224] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663884.983227] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663884.983228] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.983230] [] ? _raw_spin_lock+0x20/0x30 [2663884.983244] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.983258] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.983275] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.983294] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.983317] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.983341] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.983362] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.983366] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.983386] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.983405] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.983406] [] ? queued_spin_lock_slowpath+0xb/0xf [2663884.983408] [] ? _raw_spin_lock+0x20/0x30 [2663884.983427] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.983446] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.983447] [] ? kthread+0xd1/0xe0 [2663884.983449] [] ? insert_kthread_work+0x40/0x40 [2663884.983450] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.983451] [] ? insert_kthread_work+0x40/0x40 [2663884.983452] Task dump for CPU 23: [2663884.983453] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663884.983453] Call Trace: [2663884.983470] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663884.983487] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663884.983505] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.983524] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.983526] [] ? __wake_up+0x13/0x20 [2663884.983544] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.983563] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.983564] [] ? kthread+0xd1/0xe0 [2663884.983566] [] ? insert_kthread_work+0x40/0x40 [2663884.983567] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663884.983568] [] ? insert_kthread_work+0x40/0x40 [2663884.983697] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663884.983718] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663884.983720] CPU: 2 PID: 24598 Comm: ll_ost00_033 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663884.983721] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663884.983721] task: ffff9c2c71fe41c0 ti: ffff9c146b1cc000 task.ti: ffff9c146b1cc000 [2663884.983724] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [2663884.983724] RSP: 0018:ffff9c146b1cfb90 EFLAGS: 00000246 [2663884.983725] RAX: 0000000000000000 RBX: ffffffffffffff10 RCX: 0000000000110000 [2663884.983726] RDX: ffff9c159f3db880 RSI: 0000000000710000 RDI: ffff9c2a4a5112dc [2663884.983726] RBP: ffff9c146b1cfb90 R08: ffff9c159f25b880 R09: 0000000000000000 [2663884.983727] R10: ffff9c146aa90500 R11: ffff9c141bec8000 R12: 0000000000710000 [2663884.983727] R13: ffff9c159f3db880 R14: 0000000000110000 R15: 0000000000000000 [2663884.983728] FS: 0000000000000000(0000) GS:ffff9c159f240000(0000) knlGS:0000000000000000 [2663884.983729] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [2663884.983730] CR2: 000000c000ae5000 CR3: 000000172e636000 CR4: 00000000007607e0 [2663884.983731] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [2663884.983731] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [2663884.983731] PKRU: 00000000 [2663884.983732] Call Trace: [2663884.983734] [] queued_spin_lock_slowpath+0xb/0xf [2663884.983736] [] _raw_spin_lock+0x20/0x30 [2663884.983753] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663884.983768] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663884.983785] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663884.983805] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663884.983830] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663884.983855] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663884.983877] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663884.983881] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663884.983901] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663884.983921] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663884.983940] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663884.983959] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663884.983978] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663884.983997] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663884.983999] [] kthread+0xd1/0xe0 [2663884.984000] [] ? insert_kthread_work+0x40/0x40 [2663884.984002] [] ret_from_fork_nospec_begin+0x7/0x21 [2663884.984003] [] ? insert_kthread_work+0x40/0x40 [2663884.984017] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b8 01 00 48 03 14 c5 20 e8 d4 94 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [2663886.202606] Lustre: 23718:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585780628/real 1585780628] req@ffff9c2c42d22d00 x1660035340779328/t0(0) o104->f2-OST0014@240@gni4:15/16 lens 296/224 e 0 to 1 dl 1585780684 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [2663886.550758] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.10.32.144@o2ib2 added to recovery queue. Health = 900 [2663886.550760] LNetError: 17456:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 8 previous similar messages [2663890.408372] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [2663890.408373] LNetError: 18738:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Skipped 1 previous similar message [2663890.408377] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.10.32.142@o2ib2 (5): c: 0, oc: 0, rc: 8 [2663890.408378] LNetError: 18738:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Skipped 1 previous similar message [2663892.079624] [] ? kthread+0xd1/0xe0 [2663892.085450] [] ? insert_kthread_work+0x40/0x40 [2663892.092308] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.099682] [] ? insert_kthread_work+0x40/0x40 [2663892.106537] Task dump for CPU 8: [2663892.110531] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663892.118399] Call Trace: [2663892.121642] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.129559] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663892.137740] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.145993] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.153890] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.162053] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.169262] [] ? _raw_spin_lock+0x20/0x30 [2663892.175708] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.183544] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.191126] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.199145] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.207433] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.214503] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.222352] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.230701] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.238552] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.247179] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.254749] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.261970] [] ? _raw_spin_lock+0x20/0x30 [2663892.268430] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.275666] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.283834] [] ? kthread+0xd1/0xe0 [2663892.289671] [] ? insert_kthread_work+0x40/0x40 [2663892.296543] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.303927] [] ? insert_kthread_work+0x40/0x40 [2663892.310798] Task dump for CPU 10: [2663892.314896] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663892.322764] Call Trace: [2663892.326007] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.333903] [] ? ktime_get_ts64+0x52/0xf0 [2663892.340320] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.347775] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.355986] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.363835] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.371926] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.379057] [] ? _raw_spin_lock+0x20/0x30 [2663892.385411] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.393155] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.400636] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.408543] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.416708] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.423651] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.431369] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.439590] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.447309] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.455810] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.463268] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.471661] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.478750] [] ? _raw_spin_lock+0x20/0x30 [2663892.485082] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.492198] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.500245] [] ? kthread+0xd1/0xe0 [2663892.505959] [] ? insert_kthread_work+0x40/0x40 [2663892.512703] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.519965] [] ? insert_kthread_work+0x40/0x40 [2663892.526709] Task dump for CPU 12: [2663892.530686] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663892.538439] Call Trace: [2663892.541552] [] sched_show_task+0xa8/0x110 [2663892.548505] [] dump_cpu_task+0x39/0x70 [2663892.554565] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663892.561134] [] rcu_check_callbacks+0x442/0x730 [2663892.567879] [] ? tick_sched_do_timer+0x50/0x50 [2663892.574613] [] update_process_times+0x46/0x80 [2663892.581263] [] tick_sched_handle+0x30/0x70 [2663892.587652] [] tick_sched_timer+0x39/0x80 [2663892.593955] [] __hrtimer_run_queues+0x10e/0x270 [2663892.600776] [] hrtimer_interrupt+0xaf/0x1d0 [2663892.607251] [] local_apic_timer_interrupt+0x3b/0x60 [2663892.614419] [] smp_apic_timer_interrupt+0x43/0x60 [2663892.621412] [] apic_timer_interrupt+0x16a/0x170 [2663892.628233] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.636897] [] queued_spin_lock_slowpath+0xb/0xf [2663892.643813] [] _raw_spin_lock+0x20/0x30 [2663892.649960] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.657498] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.664770] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.672478] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.680627] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.687386] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.694923] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.703135] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.710848] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.719164] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.726596] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.733685] [] ? _raw_spin_lock+0x20/0x30 [2663892.740009] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.746943] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.754979] [] kthread+0xd1/0xe0 [2663892.760504] [] ? insert_kthread_work+0x40/0x40 [2663892.767240] [] ret_from_fork_nospec_begin+0x7/0x21 [2663892.774322] [] ? insert_kthread_work+0x40/0x40 [2663892.781055] Task dump for CPU 14: [2663892.785013] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663892.792743] Call Trace: [2663892.795857] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.803639] [] ? ktime_get_ts64+0x52/0xf0 [2663892.809953] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.817315] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.825455] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.833234] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.841274] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.848362] [] ? _raw_spin_lock+0x20/0x30 [2663892.854689] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.862402] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.869854] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.877733] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.885874] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.892815] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.900536] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.908756] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.916475] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.924982] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.932452] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.940855] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.947960] [] ? _raw_spin_lock+0x20/0x30 [2663892.954309] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.961433] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.969495] [] ? kthread+0xd1/0xe0 [2663892.975212] [] ? insert_kthread_work+0x40/0x40 [2663892.981964] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.989235] [] ? insert_kthread_work+0x40/0x40 [2663892.995987] Task dump for CPU 16: [2663892.995988] Task dump for CPU 0: [2663892.995990] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663892.995991] Call Trace: [2663892.996008] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.996010] [] ? ktime_get_ts64+0x52/0xf0 [2663892.996014] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.996029] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.996041] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.996043] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663892.996045] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.996046] [] ? _raw_spin_lock+0x20/0x30 [2663892.996061] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.996075] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.996092] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.996111] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.996136] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.996160] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.996182] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.996185] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.996205] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.996225] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.996243] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.996262] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663892.996281] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.996300] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.996301] [] ? kthread+0xd1/0xe0 [2663892.996303] [] ? insert_kthread_work+0x40/0x40 [2663892.996304] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.996306] [] ? insert_kthread_work+0x40/0x40 [2663892.996306] Task dump for CPU 2: [2663892.996308] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663892.996308] Call Trace: [2663892.996324] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.996326] [] ? ktime_get_ts64+0x52/0xf0 [2663892.996329] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.996344] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.996356] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.996358] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.996359] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.996360] [] ? _raw_spin_lock+0x20/0x30 [2663892.996374] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.996389] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.996405] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.996424] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.996448] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.996472] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.996493] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.996497] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.996517] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.996535] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.996554] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.996572] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663892.996591] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.996610] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.996612] [] ? kthread+0xd1/0xe0 [2663892.996613] [] ? insert_kthread_work+0x40/0x40 [2663892.996615] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.996616] [] ? insert_kthread_work+0x40/0x40 [2663892.996617] Task dump for CPU 4: [2663892.996618] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663892.996618] Call Trace: [2663892.996642] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.996665] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.996687] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.996690] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.996709] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.996728] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.996730] [] ? wake_up_state+0x20/0x20 [2663892.996749] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.996768] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.996769] [] kthread+0xd1/0xe0 [2663892.996771] [] ? insert_kthread_work+0x40/0x40 [2663892.996772] [] ret_from_fork_nospec_begin+0x7/0x21 [2663892.996773] [] ? insert_kthread_work+0x40/0x40 [2663892.996774] Task dump for CPU 6: [2663892.996775] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663892.996775] Call Trace: [2663892.996791] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.996793] [] ? ktime_get_ts64+0x52/0xf0 [2663892.996796] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.996812] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.996823] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.996825] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.996826] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.996827] [] ? _raw_spin_lock+0x20/0x30 [2663892.996842] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.996856] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.996873] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.996891] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.996915] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.996938] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.996960] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.996963] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.996982] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.997001] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.997003] [] ? __wake_up+0x13/0x20 [2663892.997022] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.997040] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.997042] [] ? kthread+0xd1/0xe0 [2663892.997043] [] ? insert_kthread_work+0x40/0x40 [2663892.997044] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.997046] [] ? insert_kthread_work+0x40/0x40 [2663892.997046] Task dump for CPU 8: [2663892.997047] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663892.997048] Call Trace: [2663892.997050] [] sched_show_task+0xa8/0x110 [2663892.997051] [] dump_cpu_task+0x39/0x70 [2663892.997053] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663892.997055] [] rcu_check_callbacks+0x442/0x730 [2663892.997056] [] ? tick_sched_do_timer+0x50/0x50 [2663892.997058] [] update_process_times+0x46/0x80 [2663892.997059] [] tick_sched_handle+0x30/0x70 [2663892.997061] [] tick_sched_timer+0x39/0x80 [2663892.997062] [] __hrtimer_run_queues+0x10e/0x270 [2663892.997064] [] hrtimer_interrupt+0xaf/0x1d0 [2663892.997066] [] local_apic_timer_interrupt+0x3b/0x60 [2663892.997068] [] smp_apic_timer_interrupt+0x43/0x60 [2663892.997069] [] apic_timer_interrupt+0x16a/0x170 [2663892.997071] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.997073] [] queued_spin_lock_slowpath+0xb/0xf [2663892.997074] [] _raw_spin_lock+0x20/0x30 [2663892.997088] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.997103] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.997119] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.997138] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.997162] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.997185] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.997207] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.997210] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.997230] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.997249] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.997250] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.997252] [] ? _raw_spin_lock+0x20/0x30 [2663892.997271] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.997290] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.997291] [] kthread+0xd1/0xe0 [2663892.997293] [] ? insert_kthread_work+0x40/0x40 [2663892.997294] [] ret_from_fork_nospec_begin+0x7/0x21 [2663892.997295] [] ? insert_kthread_work+0x40/0x40 [2663892.997296] Task dump for CPU 10: [2663892.997297] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663892.997297] Call Trace: [2663892.997313] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.997315] [] ? ktime_get_ts64+0x52/0xf0 [2663892.997318] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.997334] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.997345] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.997347] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.997348] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.997350] [] ? _raw_spin_lock+0x20/0x30 [2663892.997364] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.997379] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.997395] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.997414] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.997438] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.997461] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.997483] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.997486] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.997506] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.997525] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.997543] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.997545] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.997546] [] ? _raw_spin_lock+0x20/0x30 [2663892.997565] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.997583] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.997585] [] ? kthread+0xd1/0xe0 [2663892.997586] [] ? insert_kthread_work+0x40/0x40 [2663892.997587] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.997589] [] ? insert_kthread_work+0x40/0x40 [2663892.997589] Task dump for CPU 12: [2663892.997590] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663892.997591] Call Trace: [2663892.997606] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.997608] [] ? ktime_get_ts64+0x52/0xf0 [2663892.997611] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.997627] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.997638] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.997640] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.997642] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.997643] [] ? _raw_spin_lock+0x20/0x30 [2663892.997657] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.997671] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.997687] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.997706] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.997729] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.997752] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.997774] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.997777] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.997797] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.997816] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.997818] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.997819] [] ? _raw_spin_lock+0x20/0x30 [2663892.997838] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.997857] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.997858] [] ? kthread+0xd1/0xe0 [2663892.997860] [] ? insert_kthread_work+0x40/0x40 [2663892.997861] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.997862] [] ? insert_kthread_work+0x40/0x40 [2663892.997863] Task dump for CPU 14: [2663892.997864] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663892.997864] Call Trace: [2663892.997880] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.997882] [] ? ktime_get_ts64+0x52/0xf0 [2663892.997885] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.997901] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.997912] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.997914] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.997915] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.997916] [] ? _raw_spin_lock+0x20/0x30 [2663892.997930] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.997945] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.997961] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.997980] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.998004] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.998027] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.998048] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.998052] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.998071] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.998090] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.998108] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.998110] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.998111] [] ? _raw_spin_lock+0x20/0x30 [2663892.998130] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.998149] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.998150] [] ? kthread+0xd1/0xe0 [2663892.998152] [] ? insert_kthread_work+0x40/0x40 [2663892.998153] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.998154] [] ? insert_kthread_work+0x40/0x40 [2663892.998156] Task dump for CPU 0: [2663892.998157] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663892.998158] Call Trace: [2663892.998161] [] sched_show_task+0xa8/0x110 [2663892.998162] [] dump_cpu_task+0x39/0x70 [2663892.998164] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663892.998166] [] rcu_check_callbacks+0x442/0x730 [2663892.998168] [] ? tick_sched_do_timer+0x50/0x50 [2663892.998169] [] update_process_times+0x46/0x80 [2663892.998171] [] tick_sched_handle+0x30/0x70 [2663892.998172] [] tick_sched_timer+0x39/0x80 [2663892.998174] [] __hrtimer_run_queues+0x10e/0x270 [2663892.998176] [] hrtimer_interrupt+0xaf/0x1d0 [2663892.998178] [] local_apic_timer_interrupt+0x3b/0x60 [2663892.998180] [] smp_apic_timer_interrupt+0x43/0x60 [2663892.998181] [] apic_timer_interrupt+0x16a/0x170 [2663892.998183] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663892.998185] [] queued_spin_lock_slowpath+0xb/0xf [2663892.998187] [] _raw_spin_lock+0x20/0x30 [2663892.998202] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.998216] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.998233] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.998252] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.998277] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.998301] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.998323] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.998327] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.998346] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.998366] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.998385] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.998403] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663892.998422] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.998441] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.998442] [] kthread+0xd1/0xe0 [2663892.998444] [] ? insert_kthread_work+0x40/0x40 [2663892.998445] [] ret_from_fork_nospec_begin+0x7/0x21 [2663892.998447] [] ? insert_kthread_work+0x40/0x40 [2663892.998448] Task dump for CPU 2: [2663892.998449] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663892.998449] Call Trace: [2663892.998465] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.998467] [] ? ktime_get_ts64+0x52/0xf0 [2663892.998470] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.998486] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.998499] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.998501] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.998502] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.998504] [] ? _raw_spin_lock+0x20/0x30 [2663892.998518] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.998532] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.998549] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.998568] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.998592] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.998616] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.998638] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.998641] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.998660] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.998679] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.998697] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.998715] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663892.998734] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.998752] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.998754] [] ? kthread+0xd1/0xe0 [2663892.998755] [] ? insert_kthread_work+0x40/0x40 [2663892.998757] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.998758] [] ? insert_kthread_work+0x40/0x40 [2663892.998759] Task dump for CPU 4: [2663892.998760] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663892.998760] Call Trace: [2663892.998783] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.998807] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.998828] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.998832] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.998851] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.998870] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.998871] [] ? wake_up_state+0x20/0x20 [2663892.998890] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.998908] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.998910] [] kthread+0xd1/0xe0 [2663892.998911] [] ? insert_kthread_work+0x40/0x40 [2663892.998913] [] ret_from_fork_nospec_begin+0x7/0x21 [2663892.998914] [] ? insert_kthread_work+0x40/0x40 [2663892.998915] Task dump for CPU 6: [2663892.998916] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663892.998916] Call Trace: [2663892.998932] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.998933] [] ? ktime_get_ts64+0x52/0xf0 [2663892.998937] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.998952] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.998963] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.998965] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.998967] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.998968] [] ? _raw_spin_lock+0x20/0x30 [2663892.998982] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.998996] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.999013] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.999032] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.999056] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.999079] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.999101] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.999104] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.999124] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.999142] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.999144] [] ? __wake_up+0x13/0x20 [2663892.999163] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.999182] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.999183] [] ? kthread+0xd1/0xe0 [2663892.999185] [] ? insert_kthread_work+0x40/0x40 [2663892.999186] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.999187] [] ? insert_kthread_work+0x40/0x40 [2663892.999188] Task dump for CPU 8: [2663892.999189] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663892.999189] Call Trace: [2663892.999205] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.999220] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663892.999236] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.999247] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.999249] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.999250] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.999251] [] ? _raw_spin_lock+0x20/0x30 [2663892.999265] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.999280] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.999296] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.999315] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.999338] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.999361] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.999383] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.999386] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.999406] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.999425] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.999427] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.999428] [] ? _raw_spin_lock+0x20/0x30 [2663892.999447] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.999465] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.999466] [] ? kthread+0xd1/0xe0 [2663892.999468] [] ? insert_kthread_work+0x40/0x40 [2663892.999469] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.999470] [] ? insert_kthread_work+0x40/0x40 [2663892.999471] Task dump for CPU 10: [2663892.999472] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663892.999472] Call Trace: [2663892.999488] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.999490] [] ? ktime_get_ts64+0x52/0xf0 [2663892.999493] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.999509] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.999520] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.999522] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.999523] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.999524] [] ? _raw_spin_lock+0x20/0x30 [2663892.999538] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.999553] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.999569] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.999588] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.999612] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.999635] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.999657] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.999660] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.999679] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.999698] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.999716] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663892.999717] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.999719] [] ? _raw_spin_lock+0x20/0x30 [2663892.999737] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663892.999756] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663892.999758] [] ? kthread+0xd1/0xe0 [2663892.999759] [] ? insert_kthread_work+0x40/0x40 [2663892.999760] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663892.999762] [] ? insert_kthread_work+0x40/0x40 [2663892.999763] Task dump for CPU 12: [2663892.999764] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663892.999764] Call Trace: [2663892.999780] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663892.999781] [] ? ktime_get_ts64+0x52/0xf0 [2663892.999785] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663892.999800] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663892.999811] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663892.999813] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663892.999814] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.999816] [] ? _raw_spin_lock+0x20/0x30 [2663892.999830] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663892.999844] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663892.999861] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663892.999879] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663892.999904] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663892.999927] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663892.999948] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663892.999952] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663892.999971] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663892.999991] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663892.999992] [] ? queued_spin_lock_slowpath+0xb/0xf [2663892.999994] [] ? _raw_spin_lock+0x20/0x30 [2663893.000012] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.000031] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.000032] [] ? kthread+0xd1/0xe0 [2663893.000034] [] ? insert_kthread_work+0x40/0x40 [2663893.000035] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.000036] [] ? insert_kthread_work+0x40/0x40 [2663893.000037] Task dump for CPU 14: [2663893.000038] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.000038] Call Trace: [2663893.000054] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.000056] [] ? ktime_get_ts64+0x52/0xf0 [2663893.000059] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.000074] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.000086] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.000087] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.000089] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.000090] [] ? _raw_spin_lock+0x20/0x30 [2663893.000104] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.000118] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.000135] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.000154] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.000178] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.000201] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.000223] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.000226] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.000246] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.000265] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.000283] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.000285] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.000286] [] ? _raw_spin_lock+0x20/0x30 [2663893.000305] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.000323] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.000325] [] ? kthread+0xd1/0xe0 [2663893.000326] [] ? insert_kthread_work+0x40/0x40 [2663893.000327] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.000329] [] ? insert_kthread_work+0x40/0x40 [2663893.000330] Task dump for CPU 0: [2663893.000332] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.000332] Call Trace: [2663893.000349] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.000351] [] ? ktime_get_ts64+0x52/0xf0 [2663893.000355] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.000371] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.000382] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.000384] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.000386] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.000387] [] ? _raw_spin_lock+0x20/0x30 [2663893.000402] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.000416] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.000433] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.000452] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.000477] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.000500] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.000523] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.000526] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.000546] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.000566] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.000585] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.000603] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.000622] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.000641] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.000643] [] ? kthread+0xd1/0xe0 [2663893.000644] [] ? insert_kthread_work+0x40/0x40 [2663893.000645] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.000647] [] ? insert_kthread_work+0x40/0x40 [2663893.000648] Task dump for CPU 2: [2663893.000649] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.000649] Call Trace: [2663893.000665] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.000667] [] ? ktime_get_ts64+0x52/0xf0 [2663893.000670] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.000686] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.000697] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.000699] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.000701] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.000702] [] ? _raw_spin_lock+0x20/0x30 [2663893.000716] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.000730] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.000747] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.000765] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.000789] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.000812] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.000834] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.000838] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.000858] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.000877] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.000895] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.000913] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.000932] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.000951] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.000952] [] ? kthread+0xd1/0xe0 [2663893.000954] [] ? insert_kthread_work+0x40/0x40 [2663893.000955] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.000956] [] ? insert_kthread_work+0x40/0x40 [2663893.000957] Task dump for CPU 4: [2663893.000958] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.000958] Call Trace: [2663893.000981] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.001004] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.001026] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.001030] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.001049] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.001068] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.001069] [] ? wake_up_state+0x20/0x20 [2663893.001088] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.001107] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.001108] [] kthread+0xd1/0xe0 [2663893.001110] [] ? insert_kthread_work+0x40/0x40 [2663893.001111] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.001112] [] ? insert_kthread_work+0x40/0x40 [2663893.001113] Task dump for CPU 6: [2663893.001114] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.001114] Call Trace: [2663893.001130] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.001132] [] ? ktime_get_ts64+0x52/0xf0 [2663893.001135] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.001151] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.001162] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.001164] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.001166] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.001167] [] ? _raw_spin_lock+0x20/0x30 [2663893.001181] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.001195] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.001212] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.001231] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.001254] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.001278] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.001300] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.001304] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.001323] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.001342] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.001344] [] ? __wake_up+0x13/0x20 [2663893.001363] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.001382] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.001383] [] ? kthread+0xd1/0xe0 [2663893.001384] [] ? insert_kthread_work+0x40/0x40 [2663893.001386] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.001387] [] ? insert_kthread_work+0x40/0x40 [2663893.001388] Task dump for CPU 8: [2663893.001389] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663893.001389] Call Trace: [2663893.001405] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.001420] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663893.001435] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.001446] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.001448] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.001450] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.001451] [] ? _raw_spin_lock+0x20/0x30 [2663893.001465] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.001479] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.001495] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.001514] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.001538] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.001562] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.001584] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.001587] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.001607] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.001626] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.001627] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.001629] [] ? _raw_spin_lock+0x20/0x30 [2663893.001648] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.001667] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.001668] [] ? kthread+0xd1/0xe0 [2663893.001670] [] ? insert_kthread_work+0x40/0x40 [2663893.001671] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.001672] [] ? insert_kthread_work+0x40/0x40 [2663893.001673] Task dump for CPU 10: [2663893.001674] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663893.001674] Call Trace: [2663893.001690] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.001691] [] ? ktime_get_ts64+0x52/0xf0 [2663893.001695] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.001710] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.001721] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.001723] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.001725] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.001726] [] ? _raw_spin_lock+0x20/0x30 [2663893.001740] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.001755] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.001771] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.001789] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.001813] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.001836] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.001857] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.001861] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.001880] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.001899] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.001918] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.001920] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.001921] [] ? _raw_spin_lock+0x20/0x30 [2663893.001940] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.001958] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.001960] [] ? kthread+0xd1/0xe0 [2663893.001961] [] ? insert_kthread_work+0x40/0x40 [2663893.001963] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.001964] [] ? insert_kthread_work+0x40/0x40 [2663893.001965] Task dump for CPU 12: [2663893.001966] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663893.001966] Call Trace: [2663893.001982] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.001983] [] ? ktime_get_ts64+0x52/0xf0 [2663893.001987] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.002002] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.002013] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.002015] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.002017] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.002018] [] ? _raw_spin_lock+0x20/0x30 [2663893.002032] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.002046] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.002063] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.002082] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.002106] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.002129] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.002150] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.002154] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.002173] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.002192] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.002193] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.002194] [] ? _raw_spin_lock+0x20/0x30 [2663893.002213] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.002233] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.002234] [] ? kthread+0xd1/0xe0 [2663893.002235] [] ? insert_kthread_work+0x40/0x40 [2663893.002237] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.002238] [] ? insert_kthread_work+0x40/0x40 [2663893.002239] Task dump for CPU 14: [2663893.002240] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.002240] Call Trace: [2663893.002256] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.002257] [] ? ktime_get_ts64+0x52/0xf0 [2663893.002261] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.002276] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.002287] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.002289] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.002291] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.002292] [] ? _raw_spin_lock+0x20/0x30 [2663893.002306] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.002320] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.002337] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.002355] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.002379] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.002403] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.002424] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.002427] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.002447] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.002465] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.002484] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.002485] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.002486] [] ? _raw_spin_lock+0x20/0x30 [2663893.002505] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.002524] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.002525] [] ? kthread+0xd1/0xe0 [2663893.002527] [] ? insert_kthread_work+0x40/0x40 [2663893.002528] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.002529] [] ? insert_kthread_work+0x40/0x40 [2663893.002531] Task dump for CPU 0: [2663893.002533] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.002534] Call Trace: [2663893.002563] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.002566] [] ? ktime_get_ts64+0x52/0xf0 [2663893.002573] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.002588] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.002621] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.002623] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.002625] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.002627] [] ? _raw_spin_lock+0x20/0x30 [2663893.002641] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.002655] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.002672] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.002691] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.002719] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.002743] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.002765] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.002769] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.002789] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.002808] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.002827] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.002846] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.002865] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.002884] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.002886] [] ? kthread+0xd1/0xe0 [2663893.002887] [] ? insert_kthread_work+0x40/0x40 [2663893.002889] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.002890] [] ? insert_kthread_work+0x40/0x40 [2663893.002891] Task dump for CPU 2: [2663893.002893] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.002893] Call Trace: [2663893.002909] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.002911] [] ? ktime_get_ts64+0x52/0xf0 [2663893.002914] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.002930] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.002941] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.002943] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.002945] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.002946] [] ? _raw_spin_lock+0x20/0x30 [2663893.002960] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.002974] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.002991] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.003010] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.003034] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.003057] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.003079] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.003083] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.003102] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.003121] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.003139] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.003158] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.003176] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.003195] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.003196] [] ? kthread+0xd1/0xe0 [2663893.003198] [] ? insert_kthread_work+0x40/0x40 [2663893.003199] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.003201] [] ? insert_kthread_work+0x40/0x40 [2663893.003201] Task dump for CPU 4: [2663893.003203] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.003203] Call Trace: [2663893.003226] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.003250] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.003272] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.003276] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.003295] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.003314] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.003316] [] ? wake_up_state+0x20/0x20 [2663893.003335] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.003354] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.003355] [] kthread+0xd1/0xe0 [2663893.003357] [] ? insert_kthread_work+0x40/0x40 [2663893.003358] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.003360] [] ? insert_kthread_work+0x40/0x40 [2663893.003360] Task dump for CPU 6: [2663893.003362] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.003362] Call Trace: [2663893.003378] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.003379] [] ? ktime_get_ts64+0x52/0xf0 [2663893.003383] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.003399] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.003410] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.003412] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.003413] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.003414] [] ? _raw_spin_lock+0x20/0x30 [2663893.003429] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.003443] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.003459] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.003478] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.003502] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.003525] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.003546] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.003550] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.003569] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.003588] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.003590] [] ? __wake_up+0x13/0x20 [2663893.003609] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.003628] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.003629] [] ? kthread+0xd1/0xe0 [2663893.003631] [] ? insert_kthread_work+0x40/0x40 [2663893.003632] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.003633] [] ? insert_kthread_work+0x40/0x40 [2663893.003634] Task dump for CPU 8: [2663893.003635] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663893.003636] Call Trace: [2663893.003652] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.003667] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663893.003682] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.003694] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.003695] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.003697] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.003698] [] ? _raw_spin_lock+0x20/0x30 [2663893.003712] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.003726] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.003743] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.003762] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.003785] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.003809] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.003830] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.003833] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.003852] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.003871] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.003874] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.003875] [] ? _raw_spin_lock+0x20/0x30 [2663893.003894] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.003913] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.003914] [] ? kthread+0xd1/0xe0 [2663893.003916] [] ? insert_kthread_work+0x40/0x40 [2663893.003917] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.003918] [] ? insert_kthread_work+0x40/0x40 [2663893.003919] Task dump for CPU 10: [2663893.003920] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663893.003921] Call Trace: [2663893.003937] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.003938] [] ? ktime_get_ts64+0x52/0xf0 [2663893.003942] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.003957] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.003968] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.003970] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.003972] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.003973] [] ? _raw_spin_lock+0x20/0x30 [2663893.003987] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.004001] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.004018] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.004037] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.004061] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.004085] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.004106] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.004110] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.004130] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.004148] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.004167] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.004168] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.004169] [] ? _raw_spin_lock+0x20/0x30 [2663893.004188] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.004207] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.004209] [] ? kthread+0xd1/0xe0 [2663893.004210] [] ? insert_kthread_work+0x40/0x40 [2663893.004211] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.004213] [] ? insert_kthread_work+0x40/0x40 [2663893.004214] Task dump for CPU 12: [2663893.004215] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663893.004215] Call Trace: [2663893.004231] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.004233] [] ? ktime_get_ts64+0x52/0xf0 [2663893.004236] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.004252] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.004263] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.004265] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.004266] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.004268] [] ? _raw_spin_lock+0x20/0x30 [2663893.004282] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.004296] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.004313] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.004331] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.004355] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.004379] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.004400] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.004404] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.004424] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.004442] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.004444] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.004445] [] ? _raw_spin_lock+0x20/0x30 [2663893.004464] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.004483] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.004484] [] ? kthread+0xd1/0xe0 [2663893.004486] [] ? insert_kthread_work+0x40/0x40 [2663893.004487] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.004488] [] ? insert_kthread_work+0x40/0x40 [2663893.004489] Task dump for CPU 14: [2663893.004490] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.004491] Call Trace: [2663893.004506] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.004508] [] ? ktime_get_ts64+0x52/0xf0 [2663893.004511] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.004527] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.004538] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.004540] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.004541] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.004542] [] ? _raw_spin_lock+0x20/0x30 [2663893.004557] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.004571] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.004587] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.004606] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.004629] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.004652] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.004674] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.004678] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.004697] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.004716] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.004735] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.004737] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.004738] [] ? _raw_spin_lock+0x20/0x30 [2663893.004757] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.004776] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.004777] [] ? kthread+0xd1/0xe0 [2663893.004778] [] ? insert_kthread_work+0x40/0x40 [2663893.004780] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.004781] [] ? insert_kthread_work+0x40/0x40 [2663893.004782] Task dump for CPU 0: [2663893.004784] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.004785] Call Trace: [2663893.004802] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.004804] [] ? ktime_get_ts64+0x52/0xf0 [2663893.004807] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.004823] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.004835] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.004837] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.004838] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.004839] [] ? _raw_spin_lock+0x20/0x30 [2663893.004854] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.004868] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.004885] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.004904] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.004928] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.004952] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.004974] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.004977] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.004997] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.005016] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.005035] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.005054] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.005073] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.005092] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.005093] [] ? kthread+0xd1/0xe0 [2663893.005094] [] ? insert_kthread_work+0x40/0x40 [2663893.005096] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.005097] [] ? insert_kthread_work+0x40/0x40 [2663893.005098] Task dump for CPU 2: [2663893.005099] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.005100] Call Trace: [2663893.005116] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.005117] [] ? ktime_get_ts64+0x52/0xf0 [2663893.005121] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.005136] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.005148] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.005149] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.005151] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.005152] [] ? _raw_spin_lock+0x20/0x30 [2663893.005166] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.005180] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.005197] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.005216] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.005240] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.005263] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.005285] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.005289] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.005308] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.005327] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.005345] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.005363] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.005381] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.005400] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.005401] [] ? kthread+0xd1/0xe0 [2663893.005402] [] ? insert_kthread_work+0x40/0x40 [2663893.005404] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.005405] [] ? insert_kthread_work+0x40/0x40 [2663893.005406] Task dump for CPU 4: [2663893.005407] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.005407] Call Trace: [2663893.005430] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.005454] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.005476] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.005479] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.005499] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.005518] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.005520] [] ? wake_up_state+0x20/0x20 [2663893.005538] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.005557] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.005559] [] kthread+0xd1/0xe0 [2663893.005560] [] ? insert_kthread_work+0x40/0x40 [2663893.005562] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.005563] [] ? insert_kthread_work+0x40/0x40 [2663893.005564] Task dump for CPU 6: [2663893.005565] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.005565] Call Trace: [2663893.005581] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.005583] [] ? ktime_get_ts64+0x52/0xf0 [2663893.005586] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.005602] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.005613] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.005615] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.005616] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.005618] [] ? _raw_spin_lock+0x20/0x30 [2663893.005632] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.005646] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.005663] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.005681] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.005705] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.005728] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.005749] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.005753] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.005772] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.005791] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.005793] [] ? __wake_up+0x13/0x20 [2663893.005812] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.005831] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.005832] [] ? kthread+0xd1/0xe0 [2663893.005834] [] ? insert_kthread_work+0x40/0x40 [2663893.005835] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.005836] [] ? insert_kthread_work+0x40/0x40 [2663893.005837] Task dump for CPU 8: [2663893.005838] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663893.005838] Call Trace: [2663893.005854] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.005869] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663893.005884] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.005896] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.005897] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.005899] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.005900] [] ? _raw_spin_lock+0x20/0x30 [2663893.005914] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.005928] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.005945] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.005963] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.005987] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.006010] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.006031] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.006035] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.006054] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.006073] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.006074] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.006075] [] ? _raw_spin_lock+0x20/0x30 [2663893.006094] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.006113] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.006114] [] ? kthread+0xd1/0xe0 [2663893.006116] [] ? insert_kthread_work+0x40/0x40 [2663893.006117] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.006118] [] ? insert_kthread_work+0x40/0x40 [2663893.006119] Task dump for CPU 10: [2663893.006120] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663893.006120] Call Trace: [2663893.006122] [] sched_show_task+0xa8/0x110 [2663893.006124] [] dump_cpu_task+0x39/0x70 [2663893.006126] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.006127] [] rcu_check_callbacks+0x442/0x730 [2663893.006129] [] ? tick_sched_do_timer+0x50/0x50 [2663893.006131] [] update_process_times+0x46/0x80 [2663893.006132] [] tick_sched_handle+0x30/0x70 [2663893.006133] [] tick_sched_timer+0x39/0x80 [2663893.006135] [] __hrtimer_run_queues+0x10e/0x270 [2663893.006137] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.006139] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.006141] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.006142] [] apic_timer_interrupt+0x16a/0x170 [2663893.006144] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.006146] [] queued_spin_lock_slowpath+0xb/0xf [2663893.006147] [] _raw_spin_lock+0x20/0x30 [2663893.006161] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.006176] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.006192] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.006211] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.006235] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.006258] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.006280] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.006284] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.006303] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.006322] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.006340] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.006342] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.006343] [] ? _raw_spin_lock+0x20/0x30 [2663893.006361] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.006380] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.006381] [] kthread+0xd1/0xe0 [2663893.006383] [] ? insert_kthread_work+0x40/0x40 [2663893.006384] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.006385] [] ? insert_kthread_work+0x40/0x40 [2663893.006386] Task dump for CPU 12: [2663893.006387] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663893.006387] Call Trace: [2663893.006403] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.006405] [] ? ktime_get_ts64+0x52/0xf0 [2663893.006409] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.006424] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.006435] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.006437] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.006438] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.006440] [] ? _raw_spin_lock+0x20/0x30 [2663893.006454] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.006468] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.006484] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.006503] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.006527] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.006551] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.006572] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.006576] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.006595] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.006614] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.006616] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.006617] [] ? _raw_spin_lock+0x20/0x30 [2663893.006636] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.006655] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.006656] [] ? kthread+0xd1/0xe0 [2663893.006658] [] ? insert_kthread_work+0x40/0x40 [2663893.006659] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.006660] [] ? insert_kthread_work+0x40/0x40 [2663893.006661] Task dump for CPU 14: [2663893.006662] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.006662] Call Trace: [2663893.006678] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.006679] [] ? ktime_get_ts64+0x52/0xf0 [2663893.006683] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.006698] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.006710] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.006711] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.006713] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.006714] [] ? _raw_spin_lock+0x20/0x30 [2663893.006728] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.006743] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.006759] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.006778] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.006801] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.006825] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.006846] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.006850] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.006870] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.006888] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.006907] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.006908] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.006910] [] ? _raw_spin_lock+0x20/0x30 [2663893.006928] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.006947] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.006949] [] ? kthread+0xd1/0xe0 [2663893.006950] [] ? insert_kthread_work+0x40/0x40 [2663893.006951] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.006953] [] ? insert_kthread_work+0x40/0x40 [2663893.006954] Task dump for CPU 0: [2663893.006955] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.006956] Call Trace: [2663893.006973] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.006975] [] ? ktime_get_ts64+0x52/0xf0 [2663893.006979] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.006994] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.007006] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.007008] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.007009] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.007011] [] ? _raw_spin_lock+0x20/0x30 [2663893.007025] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.007039] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.007056] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.007076] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.007100] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.007124] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.007146] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.007150] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.007169] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.007189] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.007208] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.007226] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.007245] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.007264] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.007265] [] ? kthread+0xd1/0xe0 [2663893.007267] [] ? insert_kthread_work+0x40/0x40 [2663893.007268] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.007269] [] ? insert_kthread_work+0x40/0x40 [2663893.007270] Task dump for CPU 2: [2663893.007272] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.007272] Call Trace: [2663893.007287] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.007289] [] ? ktime_get_ts64+0x52/0xf0 [2663893.007293] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.007308] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.007320] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.007321] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.007323] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.007324] [] ? _raw_spin_lock+0x20/0x30 [2663893.007338] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.007352] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.007369] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.007387] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.007411] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.007435] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.007457] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.007460] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.007480] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.007499] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.007517] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.007536] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.007555] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.007574] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.007575] [] ? kthread+0xd1/0xe0 [2663893.007577] [] ? insert_kthread_work+0x40/0x40 [2663893.007578] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.007579] [] ? insert_kthread_work+0x40/0x40 [2663893.007580] Task dump for CPU 4: [2663893.007581] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.007581] Call Trace: [2663893.007605] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.007628] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.007650] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.007653] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.007673] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.007691] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.007693] [] ? wake_up_state+0x20/0x20 [2663893.007712] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.007731] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.007732] [] kthread+0xd1/0xe0 [2663893.007734] [] ? insert_kthread_work+0x40/0x40 [2663893.007735] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.007736] [] ? insert_kthread_work+0x40/0x40 [2663893.007737] Task dump for CPU 6: [2663893.007738] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.007738] Call Trace: [2663893.007754] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.007756] [] ? ktime_get_ts64+0x52/0xf0 [2663893.007759] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.007775] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.007786] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.007788] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.007789] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.007790] [] ? _raw_spin_lock+0x20/0x30 [2663893.007804] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.007819] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.007835] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.007854] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.007878] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.007901] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.007923] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.007927] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.007946] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.007965] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.007967] [] ? __wake_up+0x13/0x20 [2663893.007985] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.008004] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.008005] [] ? kthread+0xd1/0xe0 [2663893.008007] [] ? insert_kthread_work+0x40/0x40 [2663893.008008] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.008009] [] ? insert_kthread_work+0x40/0x40 [2663893.008010] Task dump for CPU 8: [2663893.008011] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663893.008011] Call Trace: [2663893.008027] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.008043] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663893.008058] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.008069] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.008071] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.008073] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008074] [] ? _raw_spin_lock+0x20/0x30 [2663893.008088] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.008102] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.008119] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.008138] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.008162] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.008185] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.008207] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.008210] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.008230] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.008249] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.008250] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008251] [] ? _raw_spin_lock+0x20/0x30 [2663893.008270] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.008289] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.008290] [] ? kthread+0xd1/0xe0 [2663893.008292] [] ? insert_kthread_work+0x40/0x40 [2663893.008293] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.008294] [] ? insert_kthread_work+0x40/0x40 [2663893.008295] Task dump for CPU 10: [2663893.008296] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663893.008296] Call Trace: [2663893.008312] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.008314] [] ? ktime_get_ts64+0x52/0xf0 [2663893.008317] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.008333] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.008344] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.008346] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.008347] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008348] [] ? _raw_spin_lock+0x20/0x30 [2663893.008362] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.008377] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.008393] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.008412] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.008435] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.008458] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.008480] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.008484] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.008503] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.008522] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.008540] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.008542] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008543] [] ? _raw_spin_lock+0x20/0x30 [2663893.008562] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.008581] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.008582] [] ? kthread+0xd1/0xe0 [2663893.008584] [] ? insert_kthread_work+0x40/0x40 [2663893.008585] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.008586] [] ? insert_kthread_work+0x40/0x40 [2663893.008587] Task dump for CPU 12: [2663893.008588] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663893.008588] Call Trace: [2663893.008604] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.008606] [] ? ktime_get_ts64+0x52/0xf0 [2663893.008609] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.008624] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.008636] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.008637] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.008639] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008640] [] ? _raw_spin_lock+0x20/0x30 [2663893.008654] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.008668] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.008685] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.008704] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.008727] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.008750] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.008772] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.008776] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.008795] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.008814] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.008815] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008816] [] ? _raw_spin_lock+0x20/0x30 [2663893.008835] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.008855] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.008856] [] ? kthread+0xd1/0xe0 [2663893.008857] [] ? insert_kthread_work+0x40/0x40 [2663893.008859] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.008860] [] ? insert_kthread_work+0x40/0x40 [2663893.008861] Task dump for CPU 14: [2663893.008862] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.008862] Call Trace: [2663893.008878] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.008879] [] ? ktime_get_ts64+0x52/0xf0 [2663893.008883] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.008898] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.008909] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.008911] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.008913] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.008914] [] ? _raw_spin_lock+0x20/0x30 [2663893.008928] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.008942] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.008959] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.008977] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.009001] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.009024] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.009046] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.009050] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.009069] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.009088] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.009106] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.009107] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.009108] [] ? _raw_spin_lock+0x20/0x30 [2663893.009127] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.009146] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.009148] [] ? kthread+0xd1/0xe0 [2663893.009149] [] ? insert_kthread_work+0x40/0x40 [2663893.009150] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.009152] [] ? insert_kthread_work+0x40/0x40 [2663893.009153] Task dump for CPU 0: [2663893.009155] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.009155] Call Trace: [2663893.009172] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.009174] [] ? ktime_get_ts64+0x52/0xf0 [2663893.009178] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.009193] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.009205] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.009207] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.009209] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.009210] [] ? _raw_spin_lock+0x20/0x30 [2663893.009225] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.009239] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.009255] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.009275] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.009299] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.009323] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.009345] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.009349] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.009369] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.009388] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.009407] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.009425] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.009444] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.009463] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.009465] [] ? kthread+0xd1/0xe0 [2663893.009466] [] ? insert_kthread_work+0x40/0x40 [2663893.009467] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.009469] [] ? insert_kthread_work+0x40/0x40 [2663893.009470] Task dump for CPU 2: [2663893.009471] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.009471] Call Trace: [2663893.009487] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.009489] [] ? ktime_get_ts64+0x52/0xf0 [2663893.009492] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.009508] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.009519] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.009521] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.009523] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.009524] [] ? _raw_spin_lock+0x20/0x30 [2663893.009538] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.009552] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.009569] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.009588] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.009612] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.009635] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.009657] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.009661] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.009680] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.009699] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.009717] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.009736] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.009754] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.009773] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.009774] [] ? kthread+0xd1/0xe0 [2663893.009776] [] ? insert_kthread_work+0x40/0x40 [2663893.009777] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.009778] [] ? insert_kthread_work+0x40/0x40 [2663893.009779] Task dump for CPU 4: [2663893.009780] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.009781] Call Trace: [2663893.009803] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.009826] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.009848] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.009851] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.009871] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.009890] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.009891] [] ? wake_up_state+0x20/0x20 [2663893.009910] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.009929] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.009930] [] kthread+0xd1/0xe0 [2663893.009932] [] ? insert_kthread_work+0x40/0x40 [2663893.009933] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.009934] [] ? insert_kthread_work+0x40/0x40 [2663893.009935] Task dump for CPU 6: [2663893.009936] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.009937] Call Trace: [2663893.009939] [] sched_show_task+0xa8/0x110 [2663893.009940] [] dump_cpu_task+0x39/0x70 [2663893.009942] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.009943] [] rcu_check_callbacks+0x442/0x730 [2663893.009945] [] ? tick_sched_do_timer+0x50/0x50 [2663893.009947] [] update_process_times+0x46/0x80 [2663893.009948] [] tick_sched_handle+0x30/0x70 [2663893.009950] [] tick_sched_timer+0x39/0x80 [2663893.009951] [] __hrtimer_run_queues+0x10e/0x270 [2663893.009953] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.009955] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.009957] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.009958] [] apic_timer_interrupt+0x16a/0x170 [2663893.009960] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.009962] [] queued_spin_lock_slowpath+0xb/0xf [2663893.009963] [] _raw_spin_lock+0x20/0x30 [2663893.009977] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.009992] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.010008] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.010027] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.010051] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.010074] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.010095] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.010099] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.010118] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.010136] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.010138] [] ? __wake_up+0x13/0x20 [2663893.010157] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.010176] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.010177] [] kthread+0xd1/0xe0 [2663893.010179] [] ? insert_kthread_work+0x40/0x40 [2663893.010180] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.010181] [] ? insert_kthread_work+0x40/0x40 [2663893.010182] Task dump for CPU 8: [2663893.010183] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663893.010183] Call Trace: [2663893.010199] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.010215] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663893.010230] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.010241] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.010243] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.010244] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.010245] [] ? _raw_spin_lock+0x20/0x30 [2663893.010260] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.010274] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.010290] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.010309] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.010333] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.010356] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.010378] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.010382] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.010401] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.010420] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.010422] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.010423] [] ? _raw_spin_lock+0x20/0x30 [2663893.010442] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.010461] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.010462] [] ? kthread+0xd1/0xe0 [2663893.010463] [] ? insert_kthread_work+0x40/0x40 [2663893.010465] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.010466] [] ? insert_kthread_work+0x40/0x40 [2663893.010467] Task dump for CPU 10: [2663893.010468] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663893.010468] Call Trace: [2663893.010484] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.010485] [] ? ktime_get_ts64+0x52/0xf0 [2663893.010489] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.010504] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.010515] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.010517] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.010518] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.010519] [] ? _raw_spin_lock+0x20/0x30 [2663893.010533] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.010548] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.010564] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.010583] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.010607] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.010630] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.010652] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.010655] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.010675] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.010694] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.010712] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.010714] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.010715] [] ? _raw_spin_lock+0x20/0x30 [2663893.010733] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.010752] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.010754] [] ? kthread+0xd1/0xe0 [2663893.010755] [] ? insert_kthread_work+0x40/0x40 [2663893.010756] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.010758] [] ? insert_kthread_work+0x40/0x40 [2663893.010758] Task dump for CPU 12: [2663893.010759] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663893.010760] Call Trace: [2663893.010775] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.010777] [] ? ktime_get_ts64+0x52/0xf0 [2663893.010780] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.010796] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.010807] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.010809] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.010810] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.010811] [] ? _raw_spin_lock+0x20/0x30 [2663893.010826] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.010840] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.010856] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.010875] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.010898] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.010922] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.010943] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.010947] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.010966] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.010986] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.010987] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.010988] [] ? _raw_spin_lock+0x20/0x30 [2663893.011008] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.011027] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.011028] [] ? kthread+0xd1/0xe0 [2663893.011030] [] ? insert_kthread_work+0x40/0x40 [2663893.011031] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.011032] [] ? insert_kthread_work+0x40/0x40 [2663893.011033] Task dump for CPU 14: [2663893.011034] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.011034] Call Trace: [2663893.011050] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.011051] [] ? ktime_get_ts64+0x52/0xf0 [2663893.011055] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.011070] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.011082] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.011083] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.011085] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.011086] [] ? _raw_spin_lock+0x20/0x30 [2663893.011100] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.011114] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.011131] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.011150] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.011173] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.011196] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.011217] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.011220] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.011240] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.011259] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.011277] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.011279] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.011280] [] ? _raw_spin_lock+0x20/0x30 [2663893.011299] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.011318] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.011320] [] ? kthread+0xd1/0xe0 [2663893.011321] [] ? insert_kthread_work+0x40/0x40 [2663893.011322] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.011324] [] ? insert_kthread_work+0x40/0x40 [2663893.011325] Task dump for CPU 0: [2663893.011327] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.011327] Call Trace: [2663893.011346] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.011348] [] ? ktime_get_ts64+0x52/0xf0 [2663893.011352] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.011368] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.011380] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.011382] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.011385] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.011386] [] ? _raw_spin_lock+0x20/0x30 [2663893.011400] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.011414] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.011431] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.011451] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.011477] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.011501] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.011523] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.011527] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.011547] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.011566] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.011586] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.011605] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.011624] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.011643] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.011645] [] ? kthread+0xd1/0xe0 [2663893.011646] [] ? insert_kthread_work+0x40/0x40 [2663893.011648] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.011649] [] ? insert_kthread_work+0x40/0x40 [2663893.011650] Task dump for CPU 2: [2663893.011651] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.011652] Call Trace: [2663893.011667] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.011669] [] ? ktime_get_ts64+0x52/0xf0 [2663893.011673] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.011688] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.011699] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.011701] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.011703] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.011704] [] ? _raw_spin_lock+0x20/0x30 [2663893.011719] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.011733] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.011750] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.011769] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.011793] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.011817] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.011839] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.011842] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.011862] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.011881] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.011899] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.011917] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.011936] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.011955] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.011956] [] ? kthread+0xd1/0xe0 [2663893.011958] [] ? insert_kthread_work+0x40/0x40 [2663893.011959] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.011960] [] ? insert_kthread_work+0x40/0x40 [2663893.011961] Task dump for CPU 4: [2663893.011962] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.011962] Call Trace: [2663893.011986] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.012009] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.012031] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.012034] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.012054] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.012072] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.012074] [] ? wake_up_state+0x20/0x20 [2663893.012093] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.012112] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.012113] [] kthread+0xd1/0xe0 [2663893.012114] [] ? insert_kthread_work+0x40/0x40 [2663893.012116] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.012117] [] ? insert_kthread_work+0x40/0x40 [2663893.012118] Task dump for CPU 6: [2663893.012119] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.012119] Call Trace: [2663893.012135] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.012137] [] ? ktime_get_ts64+0x52/0xf0 [2663893.012141] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.012156] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.012167] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.012169] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.012171] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.012172] [] ? _raw_spin_lock+0x20/0x30 [2663893.012186] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.012200] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.012217] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.012235] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.012259] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.012283] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.012305] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.012308] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.012328] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.012347] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.012348] [] ? __wake_up+0x13/0x20 [2663893.012367] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.012386] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.012388] [] ? kthread+0xd1/0xe0 [2663893.012389] [] ? insert_kthread_work+0x40/0x40 [2663893.012390] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.012392] [] ? insert_kthread_work+0x40/0x40 [2663893.012392] Task dump for CPU 8: [2663893.012394] ll_ost00_112 R running task 0 26900 2 0x00000008 [2663893.012394] Call Trace: [2663893.012410] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.012425] [] ? ldlm_extent_compat_queue+0xe2/0xa50 [ptlrpc] [2663893.012440] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.012452] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.012453] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.012455] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.012456] [] ? _raw_spin_lock+0x20/0x30 [2663893.012470] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.012485] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.012501] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.012520] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.012543] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.012566] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.012588] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.012591] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.012611] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.012629] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.012631] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.012632] [] ? _raw_spin_lock+0x20/0x30 [2663893.012651] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.012670] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.012671] [] ? kthread+0xd1/0xe0 [2663893.012673] [] ? insert_kthread_work+0x40/0x40 [2663893.012674] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.012675] [] ? insert_kthread_work+0x40/0x40 [2663893.012676] Task dump for CPU 10: [2663893.012677] ll_ost00_070 R running task 0 24739 2 0x00000008 [2663893.012677] Call Trace: [2663893.012693] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.012695] [] ? ktime_get_ts64+0x52/0xf0 [2663893.012698] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.012714] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.012725] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.012727] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.012728] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.012729] [] ? _raw_spin_lock+0x20/0x30 [2663893.012744] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.012758] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.012774] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.012793] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.012817] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.012840] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.012862] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.012866] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.012885] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.012904] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.012923] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.012924] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.012925] [] ? _raw_spin_lock+0x20/0x30 [2663893.012944] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.012963] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.012964] [] ? kthread+0xd1/0xe0 [2663893.012966] [] ? insert_kthread_work+0x40/0x40 [2663893.012967] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.012968] [] ? insert_kthread_work+0x40/0x40 [2663893.012969] Task dump for CPU 12: [2663893.012970] ll_ost00_073 R running task 0 24742 2 0x00000008 [2663893.012970] Call Trace: [2663893.012986] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.012987] [] ? ktime_get_ts64+0x52/0xf0 [2663893.012991] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.013006] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.013017] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.013019] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.013021] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.013022] [] ? _raw_spin_lock+0x20/0x30 [2663893.013036] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.013050] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.013067] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.013086] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.013110] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.013134] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.013155] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.013159] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.013178] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.013197] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.013199] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.013200] [] ? _raw_spin_lock+0x20/0x30 [2663893.013219] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.013238] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.013240] [] ? kthread+0xd1/0xe0 [2663893.013241] [] ? insert_kthread_work+0x40/0x40 [2663893.013242] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.013244] [] ? insert_kthread_work+0x40/0x40 [2663893.013244] Task dump for CPU 14: [2663893.013245] ll_ost00_089 R running task 0 24777 2 0x00000008 [2663893.013246] Call Trace: [2663893.013261] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.013263] [] ? ktime_get_ts64+0x52/0xf0 [2663893.013267] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.013282] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.013293] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.013295] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.013296] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.013298] [] ? _raw_spin_lock+0x20/0x30 [2663893.013312] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.013326] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.013342] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.013361] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.013384] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.013408] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.013430] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.013433] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.013453] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.013472] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.013490] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.013492] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.013493] [] ? _raw_spin_lock+0x20/0x30 [2663893.013512] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.013531] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.013532] [] ? kthread+0xd1/0xe0 [2663893.013534] [] ? insert_kthread_work+0x40/0x40 [2663893.013535] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.013536] [] ? insert_kthread_work+0x40/0x40 [2663893.013538] Task dump for CPU 0: [2663893.013539] ll_ost00_056 R running task 0 24709 2 0x00000008 [2663893.013540] Call Trace: [2663893.013557] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.013559] [] ? ktime_get_ts64+0x52/0xf0 [2663893.013562] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.013578] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.013590] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.013592] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663893.013594] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.013595] [] ? _raw_spin_lock+0x20/0x30 [2663893.013609] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.013623] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.013640] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.013659] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.013684] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.013708] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.013729] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.013733] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.013753] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.013773] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.013791] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.013810] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.013829] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.013848] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.013849] [] ? kthread+0xd1/0xe0 [2663893.013851] [] ? insert_kthread_work+0x40/0x40 [2663893.013852] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.013853] [] ? insert_kthread_work+0x40/0x40 [2663893.013854] Task dump for CPU 2: [2663893.013856] ll_ost00_033 R running task 0 24598 2 0x00000008 [2663893.013856] Call Trace: [2663893.013872] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.013873] [] ? ktime_get_ts64+0x52/0xf0 [2663893.013877] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.013892] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.013903] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.013905] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.013907] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.013908] [] ? _raw_spin_lock+0x20/0x30 [2663893.013922] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.013936] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.013953] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.013972] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.013995] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.014018] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.014040] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.014044] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.014063] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.014082] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.014101] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.014120] [] ? ptlrpc_grow_req_bufs+0x50/0x2a0 [ptlrpc] [2663893.014139] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.014157] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.014158] [] ? kthread+0xd1/0xe0 [2663893.014160] [] ? insert_kthread_work+0x40/0x40 [2663893.014161] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.014162] [] ? insert_kthread_work+0x40/0x40 [2663893.014163] Task dump for CPU 4: [2663893.014164] ll_ost00_123 R running task 0 26911 2 0x00000008 [2663893.014165] Call Trace: [2663893.014167] [] sched_show_task+0xa8/0x110 [2663893.014169] [] dump_cpu_task+0x39/0x70 [2663893.014170] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.014172] [] rcu_check_callbacks+0x442/0x730 [2663893.014174] [] ? tick_sched_do_timer+0x50/0x50 [2663893.014175] [] update_process_times+0x46/0x80 [2663893.014176] [] tick_sched_handle+0x30/0x70 [2663893.014178] [] tick_sched_timer+0x39/0x80 [2663893.014180] [] __hrtimer_run_queues+0x10e/0x270 [2663893.014181] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.014184] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.014185] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.014187] [] apic_timer_interrupt+0x16a/0x170 [2663893.014188] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.014190] [] queued_spin_lock_slowpath+0xb/0xf [2663893.014191] [] _raw_spin_lock+0x20/0x30 [2663893.014206] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.014220] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.014236] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.014256] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.014279] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.014302] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.014324] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.014327] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.014347] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.014366] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.014368] [] ? wake_up_state+0x20/0x20 [2663893.014387] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.014406] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.014407] [] kthread+0xd1/0xe0 [2663893.014409] [] ? insert_kthread_work+0x40/0x40 [2663893.014410] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.014412] [] ? insert_kthread_work+0x40/0x40 [2663893.014413] Task dump for CPU 6: [2663893.014414] ll_ost00_131 R running task 0 26919 2 0x00000008 [2663893.014414] Call Trace: [2663893.014430] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.014432] [] ? ktime_get_ts64+0x52/0xf0 [2663893.014435] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.014450] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.014461] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.014463] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.014465] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.014466] [] ? _raw_spin_lock+0x20/0x30 [2663893.014480] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.014494] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.014511] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.014530] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.014554] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.014577] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.014599] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.014603] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.014622] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.014640] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.014642] [] ? __wake_up+0x13/0x20 [2663893.014661] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.014680] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.014681] [] ? kthread+0xd1/0xe0 [2663893.015132] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.015133] Call Trace: [2663893.015152] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.015167] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.015183] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.015197] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.015212] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.015226] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.015240] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.015256] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.015275] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.015299] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.015323] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.015345] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.015349] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.015369] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.015388] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.015408] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.015427] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.015428] [] ? kthread+0xd1/0xe0 [2663893.015430] [] ? insert_kthread_work+0x40/0x40 [2663893.015431] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.015433] [] ? insert_kthread_work+0x40/0x40 [2663893.015433] Task dump for CPU 18: [2663893.015435] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.015435] Call Trace: [2663893.015451] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.015453] [] ? ktime_get_ts64+0x52/0xf0 [2663893.015456] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.015471] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.015484] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.015486] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.015488] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.015489] [] ? _raw_spin_lock+0x20/0x30 [2663893.015503] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.015517] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.015534] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.015553] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.015577] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.015600] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.015622] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.015626] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.015645] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.015664] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.015666] [] ? __wake_up+0x13/0x20 [2663893.015685] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.015704] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.015706] [] ? kthread+0xd1/0xe0 [2663893.015707] [] ? insert_kthread_work+0x40/0x40 [2663893.015708] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.015710] [] ? insert_kthread_work+0x40/0x40 [2663893.015710] Task dump for CPU 20: [2663893.015712] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.015712] Call Trace: [2663893.015728] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.015729] [] ? ktime_get_ts64+0x52/0xf0 [2663893.015733] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.015748] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.015760] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.015761] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.015763] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.015764] [] ? _raw_spin_lock+0x20/0x30 [2663893.015778] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.015792] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.015809] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.015827] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.015851] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.015874] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.015896] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.015899] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.015919] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.015938] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.015957] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.015958] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.015960] [] ? _raw_spin_lock+0x20/0x30 [2663893.015979] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.015998] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.015999] [] ? kthread+0xd1/0xe0 [2663893.016000] [] ? insert_kthread_work+0x40/0x40 [2663893.016002] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.016003] [] ? insert_kthread_work+0x40/0x40 [2663893.016004] Task dump for CPU 22: [2663893.016005] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.016005] Call Trace: [2663893.016021] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.016023] [] ? ktime_get_ts64+0x52/0xf0 [2663893.016026] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.016041] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.016052] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.016054] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.016056] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.016057] [] ? _raw_spin_lock+0x20/0x30 [2663893.016071] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.016085] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.016102] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.016121] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.016144] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.016168] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.016189] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.016193] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.016213] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.016231] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.016233] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.016234] [] ? _raw_spin_lock+0x20/0x30 [2663893.016253] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.016273] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.016274] [] ? kthread+0xd1/0xe0 [2663893.016275] [] ? insert_kthread_work+0x40/0x40 [2663893.016277] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.016278] [] ? insert_kthread_work+0x40/0x40 [2663893.016279] Task dump for CPU 23: [2663893.016280] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.016280] Call Trace: [2663893.016298] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.016314] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.016333] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.016351] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.016352] [] ? __wake_up+0x13/0x20 [2663893.016371] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.016389] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.016391] [] ? kthread+0xd1/0xe0 [2663893.016392] [] ? insert_kthread_work+0x40/0x40 [2663893.016393] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.016395] [] ? insert_kthread_work+0x40/0x40 [2663893.016396] Task dump for CPU 16: [2663893.016397] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.016397] Call Trace: [2663893.016413] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.016428] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.016444] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.016458] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.016472] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.016487] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.016501] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.016517] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.016536] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.016560] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.016584] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.016605] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.016609] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.016629] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.016648] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.016667] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.016687] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.016688] [] ? kthread+0xd1/0xe0 [2663893.016689] [] ? insert_kthread_work+0x40/0x40 [2663893.016691] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.016692] [] ? insert_kthread_work+0x40/0x40 [2663893.016693] Task dump for CPU 18: [2663893.016694] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.016694] Call Trace: [2663893.016710] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.016711] [] ? ktime_get_ts64+0x52/0xf0 [2663893.016715] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.016730] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.016741] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.016743] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.016745] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.016746] [] ? _raw_spin_lock+0x20/0x30 [2663893.016760] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.016774] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.016790] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.016809] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.016833] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.016857] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.016878] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.016882] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.016901] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.016920] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.016922] [] ? __wake_up+0x13/0x20 [2663893.016941] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.016960] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.016961] [] ? kthread+0xd1/0xe0 [2663893.016963] [] ? insert_kthread_work+0x40/0x40 [2663893.016964] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.016965] [] ? insert_kthread_work+0x40/0x40 [2663893.016966] Task dump for CPU 20: [2663893.016967] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.016967] Call Trace: [2663893.016983] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.016984] [] ? ktime_get_ts64+0x52/0xf0 [2663893.016988] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.017003] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.017015] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.017016] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.017018] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.017019] [] ? _raw_spin_lock+0x20/0x30 [2663893.017033] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.017047] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.017064] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.017083] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.017106] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.017130] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.017151] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.017155] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.017174] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.017193] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.017211] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.017213] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.017214] [] ? _raw_spin_lock+0x20/0x30 [2663893.017232] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.017251] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.017253] [] ? kthread+0xd1/0xe0 [2663893.017254] [] ? insert_kthread_work+0x40/0x40 [2663893.017255] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.017257] [] ? insert_kthread_work+0x40/0x40 [2663893.017257] Task dump for CPU 22: [2663893.017258] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.017259] Call Trace: [2663893.017274] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.017276] [] ? ktime_get_ts64+0x52/0xf0 [2663893.017279] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.017295] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.017306] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.017308] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.017309] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.017310] [] ? _raw_spin_lock+0x20/0x30 [2663893.017324] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.017339] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.017355] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.017373] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.017397] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.017420] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.017442] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.017445] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.017465] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.017484] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.017485] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.017487] [] ? _raw_spin_lock+0x20/0x30 [2663893.017506] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.017525] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.017526] [] ? kthread+0xd1/0xe0 [2663893.017528] [] ? insert_kthread_work+0x40/0x40 [2663893.017529] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.017530] [] ? insert_kthread_work+0x40/0x40 [2663893.017531] Task dump for CPU 23: [2663893.017532] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.017532] Call Trace: [2663893.017549] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.017565] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.017584] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.017603] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.017604] [] ? __wake_up+0x13/0x20 [2663893.017623] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.017641] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.017642] [] ? kthread+0xd1/0xe0 [2663893.017644] [] ? insert_kthread_work+0x40/0x40 [2663893.017645] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.017646] [] ? insert_kthread_work+0x40/0x40 [2663893.017647] Task dump for CPU 16: [2663893.017649] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.017649] Call Trace: [2663893.017664] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.017680] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.017695] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.017696] Lustre: f2-OST0014: Client f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) reconnecting [2663893.017712] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.017713] Lustre: Skipped 1 previous similar message [2663893.017714] Lustre: f2-OST0038: Connection restored to f2-MDT0000-mdtlov_UUID (at 10.10.33.6@o2ib2) [2663893.017729] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.017730] Lustre: Skipped 36 previous similar messages [2663893.017744] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.017758] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.017774] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.017793] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.017817] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.017841] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.017842] Lustre: f2-OST0038: deleting orphan objects from 0x0:6902706 to 0x0:6902762 [2663893.017864] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.017868] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.017888] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.017907] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.017926] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.017945] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.017947] [] ? kthread+0xd1/0xe0 [2663893.017948] [] ? insert_kthread_work+0x40/0x40 [2663893.017949] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.017951] [] ? insert_kthread_work+0x40/0x40 [2663893.017952] Task dump for CPU 18: [2663893.017953] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.017953] Call Trace: [2663893.017969] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.017971] [] ? ktime_get_ts64+0x52/0xf0 [2663893.017974] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.017990] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.018001] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.018003] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.018004] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.018005] [] ? _raw_spin_lock+0x20/0x30 [2663893.018020] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.018034] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.018050] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.018069] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.018093] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.018116] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.018137] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.018140] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.018160] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.018178] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.018180] [] ? __wake_up+0x13/0x20 [2663893.018199] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.018218] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.018220] [] ? kthread+0xd1/0xe0 [2663893.018221] [] ? insert_kthread_work+0x40/0x40 [2663893.018222] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.018223] [] ? insert_kthread_work+0x40/0x40 [2663893.018224] Task dump for CPU 20: [2663893.018226] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.018226] Call Trace: [2663893.018242] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.018243] [] ? ktime_get_ts64+0x52/0xf0 [2663893.018247] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.018262] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.018273] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.018275] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.018277] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.018278] [] ? _raw_spin_lock+0x20/0x30 [2663893.018292] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.018306] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.018323] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.018341] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.018365] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.018388] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.018410] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.018413] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.018432] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.018451] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.018469] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.018471] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.018472] [] ? _raw_spin_lock+0x20/0x30 [2663893.018491] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.018510] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.018511] [] ? kthread+0xd1/0xe0 [2663893.018512] [] ? insert_kthread_work+0x40/0x40 [2663893.018514] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.018515] [] ? insert_kthread_work+0x40/0x40 [2663893.018516] Task dump for CPU 22: [2663893.018517] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.018517] Call Trace: [2663893.018533] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.018534] [] ? ktime_get_ts64+0x52/0xf0 [2663893.018538] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.018553] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.018564] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.018566] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.018567] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.018569] [] ? _raw_spin_lock+0x20/0x30 [2663893.018583] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.018597] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.018613] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.018632] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.018656] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.018679] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.018701] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.018704] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.018724] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.018743] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.018744] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.018745] [] ? _raw_spin_lock+0x20/0x30 [2663893.018765] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.018783] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.018785] [] ? kthread+0xd1/0xe0 [2663893.018786] [] ? insert_kthread_work+0x40/0x40 [2663893.018787] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.018789] [] ? insert_kthread_work+0x40/0x40 [2663893.018789] Task dump for CPU 23: [2663893.018791] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.018791] Call Trace: [2663893.018808] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.018824] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.018843] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.018861] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.018862] [] ? __wake_up+0x13/0x20 [2663893.018881] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.018900] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.018901] [] ? kthread+0xd1/0xe0 [2663893.018902] [] ? insert_kthread_work+0x40/0x40 [2663893.018904] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.018905] [] ? insert_kthread_work+0x40/0x40 [2663893.018906] Task dump for CPU 16: [2663893.018908] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.018908] Call Trace: [2663893.018923] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.018939] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.018954] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.018969] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.018983] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.018998] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.019012] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.019028] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.019047] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.019071] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.019095] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.019117] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.019120] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.019140] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.019159] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.019179] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.019198] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.019199] [] ? kthread+0xd1/0xe0 [2663893.019201] [] ? insert_kthread_work+0x40/0x40 [2663893.019202] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.019203] [] ? insert_kthread_work+0x40/0x40 [2663893.019204] Task dump for CPU 18: [2663893.019205] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.019206] Call Trace: [2663893.019221] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.019223] [] ? ktime_get_ts64+0x52/0xf0 [2663893.019226] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.019241] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.019253] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.019254] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.019256] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.019257] [] ? _raw_spin_lock+0x20/0x30 [2663893.019272] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.019286] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.019302] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.019321] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.019345] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.019369] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.019391] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.019394] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.019414] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.019433] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.019435] [] ? __wake_up+0x13/0x20 [2663893.019454] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.019473] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.019474] [] ? kthread+0xd1/0xe0 [2663893.019476] [] ? insert_kthread_work+0x40/0x40 [2663893.019477] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.019478] [] ? insert_kthread_work+0x40/0x40 [2663893.019479] Task dump for CPU 20: [2663893.019480] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.019481] Call Trace: [2663893.019496] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.019498] [] ? ktime_get_ts64+0x52/0xf0 [2663893.019501] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.019517] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.019528] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.019530] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.019531] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.019532] [] ? _raw_spin_lock+0x20/0x30 [2663893.019547] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.019561] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.019577] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.019596] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.019620] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.019643] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.019664] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.019668] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.019688] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.019707] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.019726] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.019727] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.019728] [] ? _raw_spin_lock+0x20/0x30 [2663893.019747] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.019766] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.019768] [] ? kthread+0xd1/0xe0 [2663893.019769] [] ? insert_kthread_work+0x40/0x40 [2663893.019771] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.019772] [] ? insert_kthread_work+0x40/0x40 [2663893.019773] Task dump for CPU 22: [2663893.019774] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.019774] Call Trace: [2663893.019776] [] sched_show_task+0xa8/0x110 [2663893.019778] [] dump_cpu_task+0x39/0x70 [2663893.019779] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.019781] [] rcu_check_callbacks+0x442/0x730 [2663893.019783] [] ? tick_sched_do_timer+0x50/0x50 [2663893.019784] [] update_process_times+0x46/0x80 [2663893.019786] [] tick_sched_handle+0x30/0x70 [2663893.019787] [] tick_sched_timer+0x39/0x80 [2663893.019789] [] __hrtimer_run_queues+0x10e/0x270 [2663893.019791] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.019793] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.019795] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.019796] [] apic_timer_interrupt+0x16a/0x170 [2663893.019798] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.019799] [] queued_spin_lock_slowpath+0xb/0xf [2663893.019801] [] _raw_spin_lock+0x20/0x30 [2663893.019815] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.019829] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.019846] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.019865] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.019888] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.019911] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.019933] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.019937] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.019956] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.019975] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.019977] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.019978] [] ? _raw_spin_lock+0x20/0x30 [2663893.019998] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.020017] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.020018] [] kthread+0xd1/0xe0 [2663893.020020] [] ? insert_kthread_work+0x40/0x40 [2663893.020021] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.020023] [] ? insert_kthread_work+0x40/0x40 [2663893.020023] Task dump for CPU 23: [2663893.020025] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.020025] Call Trace: [2663893.020042] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.020058] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.020077] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.020095] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.020097] [] ? __wake_up+0x13/0x20 [2663893.020116] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.020135] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.020136] [] ? kthread+0xd1/0xe0 [2663893.020137] [] ? insert_kthread_work+0x40/0x40 [2663893.020139] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.020140] [] ? insert_kthread_work+0x40/0x40 [2663893.020141] Task dump for CPU 16: [2663893.020143] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.020143] Call Trace: [2663893.020159] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.020175] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.020190] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.020205] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.020219] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.020233] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.020248] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.020264] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.020283] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.020307] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.020330] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.020352] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.020355] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.020375] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.020395] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.020414] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.020434] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.020435] [] ? kthread+0xd1/0xe0 [2663893.020436] [] ? insert_kthread_work+0x40/0x40 [2663893.020438] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.020439] [] ? insert_kthread_work+0x40/0x40 [2663893.020440] Task dump for CPU 18: [2663893.020441] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.020441] Call Trace: [2663893.020457] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.020459] [] ? ktime_get_ts64+0x52/0xf0 [2663893.020462] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.020478] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.020489] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.020491] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.020492] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.020493] [] ? _raw_spin_lock+0x20/0x30 [2663893.020507] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.020522] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.020538] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.020557] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.020582] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.020605] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.020627] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.020630] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.020650] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.020668] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.020670] [] ? __wake_up+0x13/0x20 [2663893.020689] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.020708] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.020709] [] ? kthread+0xd1/0xe0 [2663893.020711] [] ? insert_kthread_work+0x40/0x40 [2663893.020712] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.020713] [] ? insert_kthread_work+0x40/0x40 [2663893.020714] Task dump for CPU 20: [2663893.020715] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.020716] Call Trace: [2663893.020731] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.020733] [] ? ktime_get_ts64+0x52/0xf0 [2663893.020736] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.020752] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.020763] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.020765] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.020766] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.020767] [] ? _raw_spin_lock+0x20/0x30 [2663893.020781] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.020796] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.020812] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.020831] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.020855] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.020878] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.020900] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.020904] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.020923] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.020943] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.020961] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.020963] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.020964] [] ? _raw_spin_lock+0x20/0x30 [2663893.020983] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.021001] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.021003] [] ? kthread+0xd1/0xe0 [2663893.021004] [] ? insert_kthread_work+0x40/0x40 [2663893.021005] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.021007] [] ? insert_kthread_work+0x40/0x40 [2663893.021008] Task dump for CPU 22: [2663893.021009] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.021009] Call Trace: [2663893.021025] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.021026] [] ? ktime_get_ts64+0x52/0xf0 [2663893.021030] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.021045] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.021056] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.021058] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [2663893.021060] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.021061] [] ? _raw_spin_lock+0x20/0x30 [2663893.021075] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.021089] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.021106] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.021124] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.021148] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.021149] LNetError: 18738:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.10.33.33@o2ib2 added to recovery queue. Health = 900 [2663893.021172] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.021194] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.021198] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.021217] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.021236] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.021238] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.021239] [] ? _raw_spin_lock+0x20/0x30 [2663893.021258] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.021277] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.021278] [] ? kthread+0xd1/0xe0 [2663893.021280] [] ? insert_kthread_work+0x40/0x40 [2663893.021281] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.021282] [] ? insert_kthread_work+0x40/0x40 [2663893.021283] Task dump for CPU 23: [2663893.021284] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.021284] Call Trace: [2663893.021287] [] sched_show_task+0xa8/0x110 [2663893.021288] [] dump_cpu_task+0x39/0x70 [2663893.021290] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.021292] [] rcu_check_callbacks+0x442/0x730 [2663893.021294] [] ? tick_sched_do_timer+0x50/0x50 [2663893.021296] [] update_process_times+0x46/0x80 [2663893.021298] [] tick_sched_handle+0x30/0x70 [2663893.021299] [] tick_sched_timer+0x39/0x80 [2663893.021301] [] __hrtimer_run_queues+0x10e/0x270 [2663893.021303] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.021306] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.021308] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.021309] [] apic_timer_interrupt+0x16a/0x170 [2663893.021311] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.021313] [] queued_spin_lock_slowpath+0xb/0xf [2663893.021314] [] _raw_spin_lock+0x20/0x30 [2663893.021328] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.021342] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.021359] [] ldlm_request_cancel+0x42f/0x780 [ptlrpc] [2663893.021376] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.021392] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.021411] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.021429] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.021431] [] ? __wake_up+0x13/0x20 [2663893.021449] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.021469] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.021470] [] kthread+0xd1/0xe0 [2663893.021472] [] ? insert_kthread_work+0x40/0x40 [2663893.021473] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.021474] [] ? insert_kthread_work+0x40/0x40 [2663893.021476] Task dump for CPU 16: [2663893.021477] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.021477] Call Trace: [2663893.021493] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.021508] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.021524] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.021539] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.021553] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.021567] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.021581] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.021598] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.021616] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.021640] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.021663] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.021685] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.021689] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.021708] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.021728] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.021747] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.021767] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.021768] [] ? kthread+0xd1/0xe0 [2663893.021769] [] ? insert_kthread_work+0x40/0x40 [2663893.021771] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.021772] [] ? insert_kthread_work+0x40/0x40 [2663893.021773] Task dump for CPU 18: [2663893.021774] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.021774] Call Trace: [2663893.021790] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.021792] [] ? ktime_get_ts64+0x52/0xf0 [2663893.021795] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.021811] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.021822] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.021824] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.021825] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.021826] [] ? _raw_spin_lock+0x20/0x30 [2663893.021840] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.021855] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.021871] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.021891] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.021914] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.021938] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.021959] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.021963] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.021982] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.022001] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.022003] [] ? __wake_up+0x13/0x20 [2663893.022022] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.022041] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.022042] [] ? kthread+0xd1/0xe0 [2663893.022044] [] ? insert_kthread_work+0x40/0x40 [2663893.022045] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.022046] [] ? insert_kthread_work+0x40/0x40 [2663893.022047] Task dump for CPU 20: [2663893.022048] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.022048] Call Trace: [2663893.022064] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.022066] [] ? ktime_get_ts64+0x52/0xf0 [2663893.022069] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.022084] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.022095] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.022097] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.022099] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.022100] [] ? _raw_spin_lock+0x20/0x30 [2663893.022114] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.022128] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.022145] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.022163] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.022187] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.022211] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.022233] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.022236] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.022256] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.022275] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.022293] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.022295] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.022296] [] ? _raw_spin_lock+0x20/0x30 [2663893.022315] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.022334] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.022335] [] ? kthread+0xd1/0xe0 [2663893.022336] [] ? insert_kthread_work+0x40/0x40 [2663893.022338] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.022339] [] ? insert_kthread_work+0x40/0x40 [2663893.022340] Task dump for CPU 22: [2663893.022341] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.022341] Call Trace: [2663893.022357] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.022358] [] ? ktime_get_ts64+0x52/0xf0 [2663893.022362] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.022377] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.022389] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.022390] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.022392] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.022393] [] ? _raw_spin_lock+0x20/0x30 [2663893.022407] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.022421] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.022438] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.022457] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.022480] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.022503] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.022525] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.022529] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.022548] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.022567] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.022569] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.022570] [] ? _raw_spin_lock+0x20/0x30 [2663893.022589] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.022608] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.022609] [] ? kthread+0xd1/0xe0 [2663893.022611] [] ? insert_kthread_work+0x40/0x40 [2663893.022612] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.022613] [] ? insert_kthread_work+0x40/0x40 [2663893.022614] Task dump for CPU 23: [2663893.022616] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.022616] Call Trace: [2663893.022633] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.022649] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.022668] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.022686] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.022688] [] ? __wake_up+0x13/0x20 [2663893.022706] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.022725] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.022726] [] ? kthread+0xd1/0xe0 [2663893.022727] [] ? insert_kthread_work+0x40/0x40 [2663893.022729] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.022730] [] ? insert_kthread_work+0x40/0x40 [2663893.022731] Task dump for CPU 16: [2663893.022732] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.022733] Call Trace: [2663893.022748] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.022764] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.022779] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.022794] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.022808] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.022823] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.022837] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.022853] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.022872] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.022896] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.022919] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.022941] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.022945] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.022965] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.022984] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.023003] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.023022] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.023024] [] ? kthread+0xd1/0xe0 [2663893.023025] [] ? insert_kthread_work+0x40/0x40 [2663893.023027] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.023028] [] ? insert_kthread_work+0x40/0x40 [2663893.023029] Task dump for CPU 18: [2663893.023030] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.023030] Call Trace: [2663893.023046] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.023048] [] ? ktime_get_ts64+0x52/0xf0 [2663893.023051] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.023066] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.023078] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.023079] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.023081] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.023082] [] ? _raw_spin_lock+0x20/0x30 [2663893.023096] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.023111] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.023127] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.023146] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.023170] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.023192] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.023214] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.023217] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.023237] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.023255] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.023257] [] ? __wake_up+0x13/0x20 [2663893.023276] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.023295] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.023297] [] ? kthread+0xd1/0xe0 [2663893.023298] [] ? insert_kthread_work+0x40/0x40 [2663893.023299] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.023301] [] ? insert_kthread_work+0x40/0x40 [2663893.023301] Task dump for CPU 20: [2663893.023302] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.023303] Call Trace: [2663893.023305] [] sched_show_task+0xa8/0x110 [2663893.023306] [] dump_cpu_task+0x39/0x70 [2663893.023308] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.023309] [] rcu_check_callbacks+0x442/0x730 [2663893.023311] [] ? tick_sched_do_timer+0x50/0x50 [2663893.023313] [] update_process_times+0x46/0x80 [2663893.023314] [] tick_sched_handle+0x30/0x70 [2663893.023316] [] tick_sched_timer+0x39/0x80 [2663893.023317] [] __hrtimer_run_queues+0x10e/0x270 [2663893.023319] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.023321] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.023323] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.023324] [] apic_timer_interrupt+0x16a/0x170 [2663893.023326] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.023328] [] queued_spin_lock_slowpath+0xb/0xf [2663893.023329] [] _raw_spin_lock+0x20/0x30 [2663893.023343] [] __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.023358] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.023374] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.023393] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.023418] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.023441] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.023463] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.023466] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.023486] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.023505] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.023523] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.023525] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.023526] [] ? _raw_spin_lock+0x20/0x30 [2663893.023545] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.023564] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.023565] [] kthread+0xd1/0xe0 [2663893.023567] [] ? insert_kthread_work+0x40/0x40 [2663893.023568] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.023570] [] ? insert_kthread_work+0x40/0x40 [2663893.023570] Task dump for CPU 22: [2663893.023572] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.023572] Call Trace: [2663893.023588] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.023590] [] ? ktime_get_ts64+0x52/0xf0 [2663893.023593] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.023609] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.023620] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.023622] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.023623] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.023624] [] ? _raw_spin_lock+0x20/0x30 [2663893.023639] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.023653] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.023669] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.023688] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.023712] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.023735] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.023757] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.023760] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.023780] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.023799] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.023801] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.023802] [] ? _raw_spin_lock+0x20/0x30 [2663893.023821] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.023840] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.023841] [] ? kthread+0xd1/0xe0 [2663893.023843] [] ? insert_kthread_work+0x40/0x40 [2663893.023844] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.023845] [] ? insert_kthread_work+0x40/0x40 [2663893.023846] Task dump for CPU 23: [2663893.023848] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.023848] Call Trace: [2663893.023865] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.023881] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.023900] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.023918] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.023920] [] ? __wake_up+0x13/0x20 [2663893.023938] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.023957] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.023958] [] ? kthread+0xd1/0xe0 [2663893.023960] [] ? insert_kthread_work+0x40/0x40 [2663893.023961] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.023962] [] ? insert_kthread_work+0x40/0x40 [2663893.023963] Task dump for CPU 16: [2663893.023965] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.023965] Call Trace: [2663893.023981] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.023996] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.024012] [] ? ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.024026] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.024040] [] ? ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.024055] [] ? __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.024069] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.024085] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.024104] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.024128] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.024152] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.024173] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.024177] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.024197] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.024216] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.024235] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.024255] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.024256] [] ? kthread+0xd1/0xe0 [2663893.024257] [] ? insert_kthread_work+0x40/0x40 [2663893.024259] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.024260] [] ? insert_kthread_work+0x40/0x40 [2663893.024261] Task dump for CPU 18: [2663893.024262] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.024262] Call Trace: [2663893.024278] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.024280] [] ? ktime_get_ts64+0x52/0xf0 [2663893.024283] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.024298] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.024309] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.024311] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.024313] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.024314] [] ? _raw_spin_lock+0x20/0x30 [2663893.024328] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.024343] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.024359] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.024378] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.024401] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.024425] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.024446] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.024450] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.024469] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.024488] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.024489] [] ? __wake_up+0x13/0x20 [2663893.024508] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.024526] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.024528] [] ? kthread+0xd1/0xe0 [2663893.024529] [] ? insert_kthread_work+0x40/0x40 [2663893.024531] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.024532] [] ? insert_kthread_work+0x40/0x40 [2663893.024533] Task dump for CPU 20: [2663893.024534] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.024534] Call Trace: [2663893.024550] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.024552] [] ? ktime_get_ts64+0x52/0xf0 [2663893.024555] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.024570] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.024582] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.024583] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [2663893.024585] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.024586] [] ? _raw_spin_lock+0x20/0x30 [2663893.024600] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.024614] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.024631] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.024650] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.024674] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.024697] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.024719] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.024722] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.024742] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.024761] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.024780] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.024781] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.024782] [] ? _raw_spin_lock+0x20/0x30 [2663893.024801] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.024820] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.024822] [] ? kthread+0xd1/0xe0 [2663893.024823] [] ? insert_kthread_work+0x40/0x40 [2663893.024824] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.024826] [] ? insert_kthread_work+0x40/0x40 [2663893.024826] Task dump for CPU 22: [2663893.024828] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.024828] Call Trace: [2663893.024843] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.024845] [] ? ktime_get_ts64+0x52/0xf0 [2663893.024849] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.024864] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.024875] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.024877] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.024878] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.024879] [] ? _raw_spin_lock+0x20/0x30 [2663893.024894] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.024908] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.024924] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.024943] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.024967] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.024990] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.025012] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.025016] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.025035] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.025054] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.025056] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.025057] [] ? _raw_spin_lock+0x20/0x30 [2663893.025076] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.025095] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.025096] [] ? kthread+0xd1/0xe0 [2663893.025098] [] ? insert_kthread_work+0x40/0x40 [2663893.025099] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.025100] [] ? insert_kthread_work+0x40/0x40 [2663893.025101] Task dump for CPU 23: [2663893.025103] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.025103] Call Trace: [2663893.025120] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.025137] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.025155] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.025173] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.025175] [] ? __wake_up+0x13/0x20 [2663893.025194] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.025213] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.025214] [] ? kthread+0xd1/0xe0 [2663893.025215] [] ? insert_kthread_work+0x40/0x40 [2663893.025217] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.025218] [] ? insert_kthread_work+0x40/0x40 [2663893.025219] Task dump for CPU 16: [2663893.025220] ll_ost00_092 R running task 0 24784 2 0x00000008 [2663893.025220] Call Trace: [2663893.025223] [] sched_show_task+0xa8/0x110 [2663893.025224] [] dump_cpu_task+0x39/0x70 [2663893.025226] [] rcu_dump_cpu_stacks+0x90/0xd0 [2663893.025227] [] rcu_check_callbacks+0x442/0x730 [2663893.025229] [] ? tick_sched_do_timer+0x50/0x50 [2663893.025231] [] update_process_times+0x46/0x80 [2663893.025232] [] tick_sched_handle+0x30/0x70 [2663893.025234] [] tick_sched_timer+0x39/0x80 [2663893.025235] [] __hrtimer_run_queues+0x10e/0x270 [2663893.025237] [] hrtimer_interrupt+0xaf/0x1d0 [2663893.025239] [] local_apic_timer_interrupt+0x3b/0x60 [2663893.025241] [] smp_apic_timer_interrupt+0x43/0x60 [2663893.025242] [] apic_timer_interrupt+0x16a/0x170 [2663893.025259] [] ? ldlm_extent_policy.isra.13+0x245/0x810 [ptlrpc] [2663893.025274] [] ? ldlm_extent_policy.isra.13+0x1bc/0x810 [ptlrpc] [2663893.025290] [] ldlm_process_extent_lock+0x213/0x490 [ptlrpc] [2663893.025305] [] ? ldlm_extent_compat_queue+0xa50/0xa50 [ptlrpc] [2663893.025319] [] ldlm_reprocess_queue+0x1be/0x3f0 [ptlrpc] [2663893.025333] [] __ldlm_reprocess_all+0x102/0x360 [ptlrpc] [2663893.025348] [] ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.025364] [] ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.025383] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.025407] [] tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.025431] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.025453] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.025456] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.025477] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.025496] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.025516] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.025535] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.025536] [] kthread+0xd1/0xe0 [2663893.025538] [] ? insert_kthread_work+0x40/0x40 [2663893.025539] [] ret_from_fork_nospec_begin+0x7/0x21 [2663893.025541] [] ? insert_kthread_work+0x40/0x40 [2663893.025541] Task dump for CPU 18: [2663893.025543] ll_ost00_038 R running task 0 24607 2 0x00000008 [2663893.025543] Call Trace: [2663893.025558] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.025560] [] ? ktime_get_ts64+0x52/0xf0 [2663893.025564] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.025579] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.025590] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.025592] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.025594] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.025595] [] ? _raw_spin_lock+0x20/0x30 [2663893.025609] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.025624] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.025640] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.025659] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.025683] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.025705] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.025727] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.025730] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.025749] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.025768] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.025770] [] ? __wake_up+0x13/0x20 [2663893.025789] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.025808] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.025809] [] ? kthread+0xd1/0xe0 [2663893.025811] [] ? insert_kthread_work+0x40/0x40 [2663893.025812] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.025813] [] ? insert_kthread_work+0x40/0x40 [2663893.025814] Task dump for CPU 20: [2663893.025815] ll_ost00_049 R running task 0 24692 2 0x00000008 [2663893.025815] Call Trace: [2663893.025831] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.025833] [] ? ktime_get_ts64+0x52/0xf0 [2663893.025836] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.025851] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.025863] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.025864] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.025866] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.025867] [] ? _raw_spin_lock+0x20/0x30 [2663893.025881] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.025895] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.025912] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.025931] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.025954] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.025977] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.025999] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.026002] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.026022] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.026040] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.026058] [] ? ptlrpc_server_post_idle_rqbds+0x7e/0xf0 [ptlrpc] [2663893.026060] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.026061] [] ? _raw_spin_lock+0x20/0x30 [2663893.026080] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.026098] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.026099] [] ? kthread+0xd1/0xe0 [2663893.026101] [] ? insert_kthread_work+0x40/0x40 [2663893.026102] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.026104] [] ? insert_kthread_work+0x40/0x40 [2663893.026104] Task dump for CPU 22: [2663893.026105] ll_ost00_138 R running task 0 26926 2 0x00000008 [2663893.026106] Call Trace: [2663893.026121] [] ? ldlm_extent_compat_cb+0xab/0x240 [ptlrpc] [2663893.026123] [] ? ktime_get_ts64+0x52/0xf0 [2663893.026126] [] ? ktime_get_seconds+0x25/0x40 [libcfs] [2663893.026142] [] ? ldlm_extent_compat_queue+0x1cd/0xa50 [ptlrpc] [2663893.026153] [] ? lprocfs_counter_add+0xf9/0x160 [obdclass] [2663893.026154] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [2663893.026156] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.026157] [] ? _raw_spin_lock+0x20/0x30 [2663893.026171] [] ? __ldlm_reprocess_all+0xe2/0x360 [ptlrpc] [2663893.026185] [] ? ldlm_reprocess_all+0x13/0x20 [ptlrpc] [2663893.026202] [] ? ldlm_handle_enqueue0+0x3db/0x15f0 [ptlrpc] [2663893.026221] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [2663893.026245] [] ? tgt_enqueue+0x62/0x210 [ptlrpc] [2663893.026268] [] ? tgt_request_handle+0xada/0x1570 [ptlrpc] [2663893.026289] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663893.026293] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663893.026312] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.026331] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.026333] [] ? queued_spin_lock_slowpath+0xb/0xf [2663893.026334] [] ? _raw_spin_lock+0x20/0x30 [2663893.026353] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.026372] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.026373] [] ? kthread+0xd1/0xe0 [2663893.026375] [] ? insert_kthread_work+0x40/0x40 [2663893.026376] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.026377] [] ? insert_kthread_work+0x40/0x40 [2663893.026378] Task dump for CPU 23: [2663893.026379] ldlm_cn01_046 R running task 0 57960 2 0x00000008 [2663893.026380] Call Trace: [2663893.026396] [] ? ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [2663893.026413] [] ? ldlm_cancel_handler+0x158/0x590 [ptlrpc] [2663893.026431] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663893.026450] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663893.026451] [] ? __wake_up+0x13/0x20 [2663893.026470] [] ? ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663893.026489] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663893.026490] [] ? kthread+0xd1/0xe0 [2663893.026492] [] ? insert_kthread_work+0x40/0x40 [2663893.026493] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663893.026494] [] ? insert_kthread_work+0x40/0x40 [2663893.034873] Lustre: f2-OST0014: deleting orphan objects from 0x0:6904041 to 0x0:6904097 [2663900.384101] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.10.32.146@o2ib2: 70 seconds [2663900.384103] LNet: 18738:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 33 previous similar messages [2663902.093330] NMI watchdog: Watchdog detected hard LOCKUP on cpu 11 [2663902.093361] Modules linked in: lustre(OE) lmv(OE) mdc(OE) osc(OE) lov(OE) osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) dm_round_robin dm_multipath lnet(OE) libcfs(OE) xt_comment xt_multiport zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) sd_mod sg nfsv3 nfs_acl mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_srp scsi_transport_srp scsi_tgt rpcrdma rdma_ucm ib_iser ib_umad rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi [2663902.093391] ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt crct10dif_pclmul iTCO_vendor_support crct10dif_common crc32_pclmul dell_smbios ghash_clmulni_intel dell_wmi_descriptor dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks lpc_ich i2c_i801 ahci libahci libata mei_me mei wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad pcc_cpufreq binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache mlx5_core igb crc32c_intel mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [2663902.093394] CPU: 11 PID: 9 Comm: rcu_sched Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663902.093394] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663902.093395] Call Trace: [2663902.093401] [] dump_stack+0x19/0x1b [2663902.093404] [] watchdog_overflow_callback+0x135/0x140 [2663902.093407] [] __perf_event_overflow+0x57/0x100 [2663902.093409] [] perf_event_overflow+0x14/0x20 [2663902.093412] [] handle_pmi_common+0x1a0/0x250 [2663902.093417] [] ? ioremap_page_range+0x2e8/0x480 [2663902.093420] [] ? vunmap_page_range+0x234/0x470 [2663902.093423] [] ? ghes_copy_tofrom_phys+0x116/0x210 [2663902.093424] [] intel_pmu_handle_irq+0xcf/0x1d0 [2663902.093427] [] perf_event_nmi_handler+0x31/0x50 [2663902.093429] [] nmi_handle.isra.0+0x8c/0x150 [2663902.093431] [] do_nmi+0x218/0x460 [2663902.093432] [] end_repeat_nmi+0x1e/0x81 [2663902.093435] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663902.093436] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663902.093437] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663902.093440] [] queued_spin_lock_slowpath+0xb/0xf [2663902.093441] [] _raw_spin_lock_irqsave+0x37/0x40 [2663902.093443] [] force_qs_rnp+0x54/0x170 [2663902.093445] [] ? rcu_eqs_exit+0xa0/0xa0 [2663902.093446] [] rcu_gp_fqs+0x39/0x90 [2663902.093448] [] rcu_gp_kthread+0x3bd/0x720 [2663902.093450] [] ? wake_up_atomic_t+0x30/0x30 [2663902.093452] [] ? rcu_gp_fqs+0x90/0x90 [2663902.093453] [] kthread+0xd1/0xe0 [2663902.093455] [] ? insert_kthread_work+0x40/0x40 [2663902.093457] [] ret_from_fork_nospec_begin+0x7/0x21 [2663902.093459] [] ? insert_kthread_work+0x40/0x40 [2663902.093460] Kernel panic - not syncing: Hard LOCKUP [2663902.093461] CPU: 11 PID: 9 Comm: rcu_sched Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.9.1.el7.x86_64 #1 [2663902.093462] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018 [2663902.093462] Call Trace: [2663902.093464] [] dump_stack+0x19/0x1b [2663902.093466] [] panic+0xe8/0x21f [2663902.093467] [] ? ret_from_fork_nospec_begin+0x7/0x21 [2663902.093470] [] nmi_panic+0x3f/0x40 [2663902.093472] [] watchdog_overflow_callback+0x121/0x140 [2663902.093473] [] __perf_event_overflow+0x57/0x100 [2663902.093475] [] perf_event_overflow+0x14/0x20 [2663902.093476] [] handle_pmi_common+0x1a0/0x250 [2663902.093478] [] ? ioremap_page_range+0x2e8/0x480 [2663902.093479] [] ? vunmap_page_range+0x234/0x470 [2663902.093481] [] ? ghes_copy_tofrom_phys+0x116/0x210 [2663902.093482] [] intel_pmu_handle_irq+0xcf/0x1d0 [2663902.093484] [] perf_event_nmi_handler+0x31/0x50 [2663902.093485] [] nmi_handle.isra.0+0x8c/0x150 [2663902.093487] [] do_nmi+0x218/0x460 [2663902.093488] [] end_repeat_nmi+0x1e/0x81 [2663902.093490] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663902.093491] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663902.093492] [] ? native_queued_spin_lock_slowpath+0x156/0x200 [2663902.093494] [] queued_spin_lock_slowpath+0xb/0xf [2663902.093495] [] _raw_spin_lock_irqsave+0x37/0x40 [2663902.093496] [] force_qs_rnp+0x54/0x170 [2663902.093497] [] ? rcu_eqs_exit+0xa0/0xa0 [2663902.093499] [] rcu_gp_fqs+0x39/0x90 [2663902.093500] [] rcu_gp_kthread+0x3bd/0x720 [2663902.093502] [] ? wake_up_atomic_t+0x30/0x30 [2663902.093503] [] ? rcu_gp_fqs+0x90/0x90 [2663902.093504] [] kthread+0xd1/0xe0 [2663902.093506] [] ? insert_kthread_work+0x40/0x40 [2663902.093507] [] ret_from_fork_nospec_begin+0x7/0x21 [2663902.093509] [] ? insert_kthread_work+0x40/0x40