[ 845.227413] drm [ 845.227413] mxm_wmi [ 845.227414] ahci [ 845.227414] ixgbe(OE) [ 845.227414] aesni_intel [ 845.227415] mpt3sas [ 845.227415] mlxfw [ 845.227415] dm_multipath [ 845.227416] lrw [ 845.227416] devlink [ 845.227417] libahci [ 845.227417] gf128mul [ 845.227417] dca [ 845.227418] glue_helper [ 845.227418] ablk_helper [ 845.227418] raid_class [ 845.227419] ptp [ 845.227419] drm_panel_orientation_quirks [ 845.227419] libata [ 845.227420] cryptd [ 845.227420] scsi_transport_sas [ 845.227421] pps_core [ 845.227421] wmi [ 845.227421] sunrpc [ 845.227422] dm_mirror [ 845.227422] dm_region_hash [ 845.227422] dm_log [ 845.227423] dm_mod [ 845.227423] iscsi_tcp [ 845.227423] libiscsi_tcp [ 845.227424] libiscsi [ 845.227424] scsi_transport_iscsi [ 845.227425] fuse [ 845.227425] [ 845.227427] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 845.227427] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 845.227428] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 845.227429] RIP: 0010:[] [ 845.227431] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 845.227432] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000246 [ 845.227433] RAX: 0000000000000000 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 845.227434] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 845.227435] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: 0000000000000000 [ 845.227435] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 845.227436] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 845.227437] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 845.227439] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 845.227439] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 845.227440] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 845.227441] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 845.227441] Call Trace: [ 845.227444] [] queued_spin_lock_slowpath+0xb/0xf [ 845.227445] [] _raw_spin_lock+0x30/0x40 [ 845.227451] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 845.227460] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 845.227462] [] ? wake_up_atomic_t+0x30/0x30 [ 845.227469] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 845.227494] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 845.227517] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 845.227519] [] ? check_preempt_curr+0x80/0xa0 [ 845.227540] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 845.227561] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 845.227590] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 845.227616] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 845.227617] [] ? __getnstimeofday64+0x3f/0xd0 [ 845.227641] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 845.227665] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 845.227666] [] ? __wake_up_common_lock+0x91/0xc0 [ 845.227667] [] ? sched_feat_set+0xf0/0xf0 [ 845.227690] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 845.227713] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 845.227715] [] kthread+0xd1/0xe0 [ 845.227716] [] ? insert_kthread_work+0x40/0x40 [ 845.227718] [] ret_from_fork_nospec_begin+0x21/0x21 [ 845.227719] [] ? insert_kthread_work+0x40/0x40 [ 845.227720] Code: [ 845.227720] 0d [ 845.227721] 48 [ 845.227721] 98 [ 845.227721] 83 [ 845.227722] e2 [ 845.227722] 30 [ 845.227722] 48 [ 845.227722] 81 [ 845.227723] c2 [ 845.227723] c0 [ 845.227723] b8 [ 845.227724] 01 [ 845.227724] 00 [ 845.227724] 48 [ 845.227725] 03 [ 845.227725] 14 [ 845.227725] c5 [ 845.227726] e0 [ 845.227726] 17 [ 845.227726] d5 [ 845.227727] a0 [ 845.227727] 4c [ 845.227727] 89 [ 845.227728] 02 [ 845.227728] 41 [ 845.227728] 8b [ 845.227729] 40 [ 845.227729] 08 [ 845.227730] 85 [ 845.227730] c0 [ 845.227730] 75 [ 845.227730] 0f [ 845.227731] 0f [ 845.227731] 1f [ 845.227731] 44 [ 845.227732] 00 [ 845.227732] 00 [ 845.227732] f3 [ 845.227733] 90 [ 845.227733] 41 [ 845.227733] 8b [ 845.227734] 40 [ 845.227734] 08 [ 845.227735] <85> [ 845.227735] c0 [ 845.227735] 74 [ 845.227736] f6 [ 845.227736] 4d [ 845.227736] 8b [ 845.227737] 08 [ 845.227737] 4d [ 845.227737] 85 [ 845.227738] c9 [ 845.227738] 74 [ 845.227738] 04 [ 845.227739] 41 [ 845.227739] 0f [ 845.227739] 18 [ 845.227740] 09 [ 845.227740] 8b [ 845.227740] 17 [ 845.227741] 0f [ 845.227741] b7 [ 845.227741] c2 [ 845.227741] [ 845.724115] ipmi_msghandler [ 845.727821] acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink [ 845.806149] libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 845.829766] CPU: 13 PID: 10825 Comm: mdt01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 845.844128] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 845.856454] task: ffff98322385d280 ti: ffff983225500000 task.ti: ffff983225500000 [ 845.865289] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 845.876850] RSP: 0018:ffff983225503a50 EFLAGS: 00000246 [ 845.883261] RAX: 0000000000000000 RBX: ffff9832bf340000 RCX: 0000000000690000 [ 845.891708] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 845.900157] RBP: ffff983225503a50 R08: ffff9832bf35b8c0 R09: 0000000000000000 [ 845.908603] R10: 0000000000000001 R11: 0000000000000005 R12: ffff983225503a18 [ 845.917052] R13: ffffffffa00ec9b1 R14: ffff9832255039f8 R15: ffff983225503a28 [ 845.925502] FS: 0000000000000000(0000) GS:ffff9832bf340000(0000) knlGS:0000000000000000 [ 845.935015] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 845.941912] CR2: 00002aaaab176a00 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 845.950359] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 845.958806] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 845.967253] Call Trace: [ 845.970469] [] queued_spin_lock_slowpath+0xb/0xf [ 845.977938] [] _raw_spin_lock+0x30/0x40 [ 845.984550] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 845.992229] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 846.000677] [] ? wake_up_atomic_t+0x30/0x30 [ 846.007678] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 846.015082] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 846.023359] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 846.031909] [] ? enqueue_task_fair+0x208/0x6c0 [ 846.039184] [] ? check_preempt_curr+0x80/0xa0 [ 846.046372] [] ? ttwu_do_wakeup+0x19/0x100 [ 846.053300] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 846.061486] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 846.070421] [] ? __getnstimeofday64+0x3f/0xd0 [ 846.077633] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 846.086688] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 846.094750] [] ? __wake_up_common_lock+0x91/0xc0 [ 846.102228] [] ? sched_feat_set+0xf0/0xf0 [ 846.109053] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 846.116556] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 846.125295] [] kthread+0xd1/0xe0 [ 846.131226] [] ? insert_kthread_work+0x40/0x40 [ 846.138502] [] ret_from_fork_nospec_begin+0x21/0x21 [ 846.146272] [] ? insert_kthread_work+0x40/0x40 [ 846.153555] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 847.776293] Lustre: ls1-MDT0000: Export ffff98320f916c00 already connecting from 192.168.128.19@o2ib42 [ 848.582768] LustreError: dumping log to /tmp/lustre-log.1644342504.17318 [ 855.765938] Lustre: MGS: haven't heard from client 3fe29da5-f736-ae73-cff3-1d0ffe883bac (at 192.168.128.19@o2ib42) in 181 seconds. I think it's dead, and I am evicting it. exp ffff97f2bef35000, cur 1644342512 expire 1644342362 last 1644342331 [ 861.161203] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [mdt01_022:17601] [ 861.192206] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [mdt01_007:17310] [ 861.170232] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac [ 861.192207] Modules linked in: [ 861.192207] osp(OE) [ 861.192208] mdd(OE) [ 861.192208] lod(OE) [ 861.192209] mdt(OE) [ 861.192209] lfsck(OE) [ 861.192209] mgs(OE) [ 861.192210] mgc(OE) [ 861.192210] osd_zfs(OE) [ 861.192210] lquota(OE) [ 861.192210] fid(OE) [ 861.192211] fld(OE) [ 861.192211] ptlrpc(OE) [ 861.192211] obdclass(OE) [ 861.192212] ko2iblnd(OE) [ 861.192212] lnet(OE) [ 861.192212] libcfs(OE) [ 861.192213] iTCO_wdt [ 861.192213] iTCO_vendor_support [ 861.192213] sb_edac [ 861.192213] intel_powerclamp [ 861.192214] coretemp [ 861.192214] intel_rapl [ 861.192214] iosf_mbi [ 861.192215] rpcrdma [ 861.192215] ib_iser [ 861.192215] kvm [ 861.192215] irqbypass [ 861.192216] joydev [ 861.192216] pcspkr [ 861.192217] lpc_ich [ 861.192217] i2c_i801 [ 861.192217] ioatdma [ 861.192218] ses [ 861.192218] enclosure [ 861.192218] sg [ 861.192219] ipmi_si [ 861.192219] ipmi_devintf [ 861.192219] ipmi_msghandler [ 861.192220] acpi_power_meter [ 861.192220] dm_round_robin [ 861.192220] acpi_cpufreq [ 861.192221] ib_ipoib [ 861.192221] rdma_ucm [ 861.192221] ib_umad [ 861.192221] sch_fq_codel [ 861.192222] iw_cxgb4 [ 861.192222] rdma_cm [ 861.192222] iw_cm [ 861.192223] ib_cm [ 861.192223] iw_cxgb3 [ 861.192223] zfs(POE) [ 861.192224] binfmt_misc [ 861.192224] zunicode(POE) [ 861.192224] zavl(POE) [ 861.192225] icp(POE) [ 861.192225] zcommon(POE) [ 861.192225] znvpair(POE) [ 861.192226] spl(OE) [ 861.192226] msr_safe(OE) [ 861.192226] ip_tables [ 861.192226] nfsv3 [ 861.192227] nfs_acl [ 861.192227] rpcsec_gss_krb5 [ 861.192228] auth_rpcgss [ 861.192228] nfsv4 [ 861.192228] dns_resolver [ 861.192229] nfs [ 861.192229] lockd [ 861.192229] grace [ 861.192229] fscache [ 861.192230] overlay(T) [ 861.192230] ext4 [ 861.192231] mbcache [ 861.192231] jbd2 [ 861.192231] dm_service_time [ 861.192232] sd_mod [ 861.192232] crc_t10dif [ 861.192232] crct10dif_generic [ 861.192233] mlx5_ib [ 861.192233] ib_uverbs [ 861.192233] ib_core [ 861.192234] be2iscsi [ 861.192234] bnx2i [ 861.192235] cnic [ 861.192235] uio [ 861.192235] cxgb4i [ 861.192236] cxgb4 [ 861.192236] cxgb3i [ 861.192236] cxgb3 [ 861.192237] mdio [ 861.192237] libcxgbi [ 861.192237] libcxgb [ 861.192238] qla4xxx [ 861.192238] iscsi_boot_sysfs [ 861.192239] mgag200 [ 861.192239] i2c_algo_bit [ 861.192239] 8021q [ 861.192240] drm_kms_helper [ 861.192240] garp [ 861.192241] syscopyarea [ 861.192241] mrp [ 861.192242] crct10dif_pclmul [ 861.192242] stp [ 861.192242] sysfillrect [ 861.192243] crct10dif_common [ 861.192243] sysimgblt [ 861.192244] crc32_pclmul [ 861.192244] llc [ 861.192244] fb_sys_fops [ 861.192244] crc32c_intel [ 861.192245] ttm [ 861.192245] ghash_clmulni_intel [ 861.192245] mlx5_core [ 861.192246] drm [ 861.192246] mxm_wmi [ 861.192246] ahci [ 861.192246] ixgbe(OE) [ 861.192247] aesni_intel [ 861.192247] mpt3sas [ 861.192247] mlxfw [ 861.192248] dm_multipath [ 861.192248] lrw [ 861.192248] devlink [ 861.192249] libahci [ 861.192249] gf128mul [ 861.192249] dca [ 861.192250] glue_helper [ 861.192250] ablk_helper [ 861.192250] raid_class [ 861.192251] ptp [ 861.192251] drm_panel_orientation_quirks [ 861.192251] libata [ 861.192252] cryptd [ 861.192252] scsi_transport_sas [ 861.192253] pps_core [ 861.192253] wmi [ 861.192253] sunrpc [ 861.192254] dm_mirror [ 861.192254] dm_region_hash [ 861.192254] dm_log [ 861.192254] dm_mod [ 861.192255] iscsi_tcp [ 861.192255] libiscsi_tcp [ 861.192255] libiscsi [ 861.192256] scsi_transport_iscsi [ 861.192256] fuse [ 861.192257] [ 861.192259] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 861.192259] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 861.192260] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 861.192261] RIP: 0010:[] [ 861.192265] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 861.192266] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 861.192266] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 861.192267] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 861.192268] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 861.192268] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 861.192269] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 861.192270] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 861.192271] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 861.192271] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 861.192272] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 861.192273] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 861.192273] Call Trace: [ 861.192276] [] queued_spin_lock_slowpath+0xb/0xf [ 861.192278] [] _raw_spin_lock+0x30/0x40 [ 861.192286] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 861.192298] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 861.192300] [] ? wake_up_atomic_t+0x30/0x30 [ 861.192306] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 861.192334] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 861.192352] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 861.192354] [] ? check_preempt_curr+0x80/0xa0 [ 861.192371] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 861.192388] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 861.192413] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 861.192434] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 861.192436] [] ? __getnstimeofday64+0x3f/0xd0 [ 861.192456] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 861.192475] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 861.192476] [] ? __wake_up_common_lock+0x91/0xc0 [ 861.192477] [] ? sched_feat_set+0xf0/0xf0 [ 861.192495] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 861.192514] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 861.192515] [] kthread+0xd1/0xe0 [ 861.192517] [] ? insert_kthread_work+0x40/0x40 [ 861.192518] [] ret_from_fork_nospec_begin+0x21/0x21 [ 861.192520] [] ? insert_kthread_work+0x40/0x40 [ 861.192520] Code: [ 861.192521] 13 [ 861.192521] 48 [ 861.192521] c1 [ 861.192521] ea [ 861.192522] 0d [ 861.192522] 48 [ 861.192522] 98 [ 861.192522] 83 [ 861.192523] e2 [ 861.192523] 30 [ 861.192523] 48 [ 861.192523] 81 [ 861.192524] c2 [ 861.192524] c0 [ 861.192524] b8 [ 861.192525] 01 [ 861.192525] 00 [ 861.192525] 48 [ 861.192525] 03 [ 861.192526] 14 [ 861.192526] c5 [ 861.192526] e0 [ 861.192526] 17 [ 861.192527] d5 [ 861.192527] a0 [ 861.192527] 4c [ 861.192527] 89 [ 861.192528] 02 [ 861.192528] 41 [ 861.192528] 8b [ 861.192529] 40 [ 861.192529] 08 [ 861.192529] 85 [ 861.192530] c0 [ 861.192530] 75 [ 861.192530] 0f [ 861.192530] 0f [ 861.192531] 1f [ 861.192531] 44 [ 861.192531] 00 [ 861.192531] 00 [ 861.192532] f3 [ 861.192532] 90 [ 861.192532] <41> [ 861.192532] 8b [ 861.192533] 40 [ 861.192533] 08 [ 861.192533] 85 [ 861.192533] c0 [ 861.192534] 74 [ 861.192534] f6 [ 861.192534] 4d [ 861.192534] 8b [ 861.192535] 08 [ 861.192535] 4d [ 861.192535] 85 [ 861.192535] c9 [ 861.192536] 74 [ 861.192536] 04 [ 861.192536] 41 [ 861.192536] 0f [ 861.192537] 18 [ 861.192537] 09 [ 861.192537] 8b [ 861.192537] [ 861.206208] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ll_mgs_0013:10822] [ 861.206209] Modules linked in: [ 861.206210] osp(OE) [ 861.206211] mdd(OE) [ 861.206211] lod(OE) [ 861.206212] mdt(OE) [ 861.206212] lfsck(OE) [ 861.206213] mgs(OE) [ 861.206214] mgc(OE) [ 861.206214] osd_zfs(OE) [ 861.206215] lquota(OE) [ 861.206215] fid(OE) [ 861.206216] fld(OE) [ 861.206216] ptlrpc(OE) [ 861.206217] obdclass(OE) [ 861.206218] ko2iblnd(OE) [ 861.206218] lnet(OE) [ 861.206219] libcfs(OE) [ 861.206219] iTCO_wdt [ 861.206220] iTCO_vendor_support [ 861.206220] sb_edac [ 861.206221] intel_powerclamp [ 861.206222] coretemp [ 861.206222] intel_rapl [ 861.206223] iosf_mbi [ 861.206223] rpcrdma [ 861.206224] ib_iser [ 861.206225] kvm [ 861.206226] irqbypass [ 861.206226] joydev [ 861.206227] pcspkr [ 861.206227] lpc_ich [ 861.206228] i2c_i801 [ 861.206229] ioatdma [ 861.206229] ses [ 861.206230] enclosure [ 861.206230] sg [ 861.206231] ipmi_si [ 861.206232] ipmi_devintf [ 861.206232] ipmi_msghandler [ 861.206233] acpi_power_meter [ 861.206234] dm_round_robin [ 861.206234] acpi_cpufreq [ 861.206235] ib_ipoib [ 861.206235] rdma_ucm [ 861.206236] ib_umad [ 861.206237] sch_fq_codel [ 861.206237] iw_cxgb4 [ 861.206238] rdma_cm [ 861.206238] iw_cm [ 861.206239] ib_cm [ 861.206239] iw_cxgb3 [ 861.206240] zfs(POE) [ 861.206241] binfmt_misc [ 861.206241] zunicode(POE) [ 861.206242] zavl(POE) [ 861.206242] icp(POE) [ 861.206243] zcommon(POE) [ 861.206243] znvpair(POE) [ 861.206244] spl(OE) [ 861.206245] msr_safe(OE) [ 861.206245] ip_tables [ 861.206246] nfsv3 [ 861.206247] nfs_acl [ 861.206247] rpcsec_gss_krb5 [ 861.206248] auth_rpcgss [ 861.206248] nfsv4 [ 861.206249] dns_resolver [ 861.206249] nfs [ 861.206250] lockd [ 861.206250] grace [ 861.206251] fscache [ 861.206252] overlay(T) [ 861.206253] ext4 [ 861.206254] mbcache [ 861.206255] jbd2 [ 861.206255] dm_service_time [ 861.206256] sd_mod [ 861.206257] crc_t10dif [ 861.206257] crct10dif_generic [ 861.206258] mlx5_ib [ 861.206258] ib_uverbs [ 861.206259] ib_core [ 861.206260] be2iscsi [ 861.206260] bnx2i [ 861.206261] cnic [ 861.206261] uio [ 861.206262] cxgb4i [ 861.206263] cxgb4 [ 861.206263] cxgb3i [ 861.206264] cxgb3 [ 861.206264] mdio [ 861.206265] libcxgbi [ 861.206266] libcxgb [ 861.206266] qla4xxx [ 861.206267] iscsi_boot_sysfs [ 861.206267] mgag200 [ 861.206268] i2c_algo_bit [ 861.206269] 8021q [ 861.206269] drm_kms_helper [ 861.206270] garp [ 861.206270] syscopyarea [ 861.206271] mrp [ 861.206272] crct10dif_pclmul [ 861.206272] stp [ 861.206273] sysfillrect [ 861.206273] crct10dif_common [ 861.206274] sysimgblt [ 861.206275] crc32_pclmul [ 861.206275] llc [ 861.206276] fb_sys_fops [ 861.206277] crc32c_intel [ 861.206277] ttm [ 861.206278] ghash_clmulni_intel [ 861.206278] mlx5_core [ 861.206279] drm [ 861.206279] mxm_wmi [ 861.206280] ahci [ 861.206281] ixgbe(OE) [ 861.206281] aesni_intel [ 861.206282] mpt3sas [ 861.206283] mlxfw [ 861.206283] dm_multipath [ 861.206284] lrw [ 861.206284] devlink [ 861.206285] libahci [ 861.206285] gf128mul [ 861.206286] dca [ 861.206286] glue_helper [ 861.206287] ablk_helper [ 861.206287] raid_class [ 861.206288] ptp [ 861.206289] drm_panel_orientation_quirks [ 861.206289] libata [ 861.206290] cryptd [ 861.206291] scsi_transport_sas [ 861.206291] pps_core [ 861.206292] wmi [ 861.206292] sunrpc [ 861.206293] dm_mirror [ 861.206294] dm_region_hash [ 861.206294] dm_log [ 861.206295] dm_mod [ 861.206296] iscsi_tcp [ 861.206296] libiscsi_tcp [ 861.206297] libiscsi [ 861.206297] scsi_transport_iscsi [ 861.206298] fuse [ 861.206298] [ 861.206301] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 861.206302] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 861.206303] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 861.206304] RIP: 0010:[] [ 861.206310] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 861.206311] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 861.206312] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 861.206313] RDX: ffff9832bf35b8c0 RSI: 0000000000690001 RDI: ffff97f2a83bcf40 [ 861.206313] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 861.206314] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 861.206315] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 861.206317] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 861.206318] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 861.206319] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 861.206320] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 861.206321] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 861.206321] Call Trace: [ 861.206327] [] queued_spin_lock_slowpath+0xb/0xf [ 861.206331] [] _raw_spin_lock+0x30/0x40 [ 861.206348] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 861.206365] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 861.206368] [] ? wake_up_atomic_t+0x30/0x30 [ 861.206376] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 861.206430] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 861.206465] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 861.206468] [] ? enqueue_task_fair+0x208/0x6c0 [ 861.206470] [] ? check_preempt_curr+0x80/0xa0 [ 861.206472] [] ? ttwu_do_wakeup+0x19/0x100 [ 861.206522] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 861.206562] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 861.206564] [] ? __getnstimeofday64+0x3f/0xd0 [ 861.206602] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 861.206639] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 861.206641] [] ? __wake_up_common_lock+0x91/0xc0 [ 861.206643] [] ? sched_feat_set+0xf0/0xf0 [ 861.206681] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 861.206684] [] ? __switch_to+0xce/0x5a0 [ 861.206722] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 861.206724] [] kthread+0xd1/0xe0 [ 861.206726] [] ? insert_kthread_work+0x40/0x40 [ 861.206729] [] ret_from_fork_nospec_begin+0x21/0x21 [ 861.206730] [] ? insert_kthread_work+0x40/0x40 [ 861.206731] Code: [ 861.206732] 13 [ 861.206732] 48 [ 861.206733] c1 [ 861.206733] ea [ 861.206733] 0d [ 861.206734] 48 [ 861.206734] 98 [ 861.206735] 83 [ 861.206735] e2 [ 861.206735] 30 [ 861.206736] 48 [ 861.206736] 81 [ 861.206736] c2 [ 861.206737] c0 [ 861.206737] b8 [ 861.206738] 01 [ 861.206738] 00 [ 861.206738] 48 [ 861.206739] 03 [ 861.206739] 14 [ 861.206739] c5 [ 861.206740] e0 [ 861.206740] 17 [ 861.206740] d5 [ 861.206741] a0 [ 861.206741] 4c [ 861.206741] 89 [ 861.206742] 02 [ 861.206742] 41 [ 861.206743] 8b [ 861.206743] 40 [ 861.206743] 08 [ 861.206744] 85 [ 861.206744] c0 [ 861.206744] 75 [ 861.206745] 0f [ 861.206745] 0f [ 861.206745] 1f [ 861.206746] 44 [ 861.206746] 00 [ 861.206746] 00 [ 861.206747] f3 [ 861.206747] 90 [ 861.206748] <41> [ 861.206748] 8b [ 861.206748] 40 [ 861.206749] 08 [ 861.206749] 85 [ 861.206749] c0 [ 861.206750] 74 [ 861.206750] f6 [ 861.206750] 4d [ 861.206751] 8b [ 861.206751] 08 [ 861.206751] 4d [ 861.206752] 85 [ 861.206752] c9 [ 861.206752] 74 [ 861.206753] 04 [ 861.206753] 41 [ 861.206753] 0f [ 861.206754] 18 [ 861.206754] 09 [ 861.206754] 8b [ 861.206755] [ 861.210209] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [mdt00_009:17321] [ 861.210209] Modules linked in: [ 861.210210] osp(OE) [ 861.210210] mdd(OE) [ 861.210211] lod(OE) [ 861.210211] mdt(OE) [ 861.210211] lfsck(OE) [ 861.210212] mgs(OE) [ 861.210212] mgc(OE) [ 861.210212] osd_zfs(OE) [ 861.210213] lquota(OE) [ 861.210213] fid(OE) [ 861.210213] fld(OE) [ 861.210213] ptlrpc(OE) [ 861.210214] obdclass(OE) [ 861.210214] ko2iblnd(OE) [ 861.210214] lnet(OE) [ 861.210215] libcfs(OE) [ 861.210215] iTCO_wdt [ 861.210215] iTCO_vendor_support [ 861.210216] sb_edac [ 861.210216] intel_powerclamp [ 861.210216] coretemp [ 861.210217] intel_rapl [ 861.210217] iosf_mbi [ 861.210217] rpcrdma [ 861.210217] ib_iser [ 861.210218] kvm [ 861.210218] irqbypass [ 861.210218] joydev [ 861.210219] pcspkr [ 861.210219] lpc_ich [ 861.210219] i2c_i801 [ 861.210219] ioatdma [ 861.210220] ses [ 861.210220] enclosure [ 861.210220] sg [ 861.210220] ipmi_si [ 861.210221] ipmi_devintf [ 861.210221] ipmi_msghandler [ 861.210221] acpi_power_meter [ 861.210222] dm_round_robin [ 861.210222] acpi_cpufreq [ 861.210222] ib_ipoib [ 861.210223] rdma_ucm [ 861.210223] ib_umad [ 861.210223] sch_fq_codel [ 861.210223] iw_cxgb4 [ 861.210224] rdma_cm [ 861.210224] iw_cm [ 861.210224] ib_cm [ 861.210224] iw_cxgb3 [ 861.210225] zfs(POE) [ 861.210225] binfmt_misc [ 861.210226] zunicode(POE) [ 861.210226] zavl(POE) [ 861.210226] icp(POE) [ 861.210227] zcommon(POE) [ 861.210227] znvpair(POE) [ 861.210227] spl(OE) [ 861.210228] msr_safe(OE) [ 861.210228] ip_tables [ 861.210228] nfsv3 [ 861.210228] nfs_acl [ 861.210229] rpcsec_gss_krb5 [ 861.210229] auth_rpcgss [ 861.210229] nfsv4 [ 861.210230] dns_resolver [ 861.210230] nfs [ 861.210230] lockd [ 861.210231] grace [ 861.210231] fscache [ 861.210231] overlay(T) [ 861.210231] ext4 [ 861.210232] mbcache [ 861.210232] jbd2 [ 861.210232] dm_service_time [ 861.210233] sd_mod [ 861.210233] crc_t10dif [ 861.210233] crct10dif_generic [ 861.210233] mlx5_ib [ 861.210234] ib_uverbs [ 861.210234] ib_core [ 861.210234] be2iscsi [ 861.210235] bnx2i [ 861.210235] cnic [ 861.210235] uio [ 861.210235] cxgb4i [ 861.210236] cxgb4 [ 861.210236] cxgb3i [ 861.210236] cxgb3 [ 861.210236] mdio [ 861.210237] libcxgbi [ 861.210237] libcxgb [ 861.210237] qla4xxx [ 861.210238] iscsi_boot_sysfs [ 861.210238] mgag200 [ 861.210238] i2c_algo_bit [ 861.210238] 8021q [ 861.210239] drm_kms_helper [ 861.210239] garp [ 861.210239] syscopyarea [ 861.210240] mrp [ 861.210240] crct10dif_pclmul [ 861.210240] stp [ 861.210240] sysfillrect [ 861.210241] crct10dif_common [ 861.210241] sysimgblt [ 861.210241] crc32_pclmul [ 861.210242] llc [ 861.210242] fb_sys_fops [ 861.210242] crc32c_intel [ 861.210242] ttm [ 861.210243] ghash_clmulni_intel [ 861.210243] mlx5_core [ 861.210243] drm [ 861.210243] mxm_wmi [ 861.210244] ahci [ 861.210244] ixgbe(OE) [ 861.210244] aesni_intel [ 861.210245] mpt3sas [ 861.210245] mlxfw [ 861.210245] dm_multipath [ 861.210245] lrw [ 861.210246] devlink [ 861.210246] libahci [ 861.210246] gf128mul [ 861.210246] dca [ 861.210247] glue_helper [ 861.210247] ablk_helper [ 861.210247] raid_class [ 861.210248] ptp [ 861.210248] drm_panel_orientation_quirks [ 861.210248] libata [ 861.210248] cryptd [ 861.210249] scsi_transport_sas [ 861.210249] pps_core [ 861.210249] wmi [ 861.210250] sunrpc [ 861.210250] dm_mirror [ 861.210250] dm_region_hash [ 861.210250] dm_log [ 861.210251] dm_mod [ 861.210251] iscsi_tcp [ 861.210251] libiscsi_tcp [ 861.210252] libiscsi [ 861.210252] scsi_transport_iscsi [ 861.210252] fuse [ 861.210252] [ 861.210254] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 861.210255] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 861.210255] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 861.210256] RIP: 0010:[] [ 861.210258] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 861.210259] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 861.210259] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 861.210260] RDX: ffff9832bf59b8c0 RSI: 0000000000f10000 RDI: ffff97f2a83bcf40 [ 861.210260] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 861.210261] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 861.210262] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 861.210263] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 861.210264] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 861.210264] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 861.210265] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 861.210265] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 861.210266] Call Trace: [ 861.210268] [] queued_spin_lock_slowpath+0xb/0xf [ 861.210270] [] _raw_spin_lock+0x30/0x40 [ 861.210275] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 861.210282] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 861.210283] [] ? wake_up_atomic_t+0x30/0x30 [ 861.210289] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 861.210308] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 861.210324] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 861.210342] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 861.210344] [] ? enqueue_task_fair+0x208/0x6c0 [ 861.210345] [] ? sched_clock_cpu+0xa5/0xe0 [ 861.210347] [] ? check_preempt_curr+0x80/0xa0 [ 861.210348] [] ? ttwu_do_wakeup+0x19/0x100 [ 861.210375] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 861.210402] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 861.210404] [] ? __getnstimeofday64+0x3f/0xd0 [ 861.210430] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 861.210455] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 861.210456] [] ? __wake_up_common_lock+0x91/0xc0 [ 861.210457] [] ? sched_feat_set+0xf0/0xf0 [ 861.210482] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 861.210484] [] ? __switch_to+0xce/0x5a0 [ 861.210509] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 861.210511] [] kthread+0xd1/0xe0 [ 861.210513] [] ? insert_kthread_work+0x40/0x40 [ 861.210514] [] ret_from_fork_nospec_begin+0x21/0x21 [ 861.210515] [] ? insert_kthread_work+0x40/0x40 [ 861.210516] Code: [ 861.210516] 13 [ 861.210517] 48 [ 861.210517] c1 [ 861.210517] ea [ 861.210517] 0d [ 861.210518] 48 [ 861.210518] 98 [ 861.210518] 83 [ 861.210518] e2 [ 861.210519] 30 [ 861.210519] 48 [ 861.210519] 81 [ 861.210519] c2 [ 861.210520] c0 [ 861.210520] b8 [ 861.210520] 01 [ 861.210520] 00 [ 861.210521] 48 [ 861.210521] 03 [ 861.210521] 14 [ 861.210522] c5 [ 861.210522] e0 [ 861.210522] 17 [ 861.210522] d5 [ 861.210523] a0 [ 861.210523] 4c [ 861.210523] 89 [ 861.210523] 02 [ 861.210524] 41 [ 861.210524] 8b [ 861.210524] 40 [ 861.210524] 08 [ 861.210525] 85 [ 861.210525] c0 [ 861.210525] 75 [ 861.210525] 0f [ 861.210526] 0f [ 861.210526] 1f [ 861.210526] 44 [ 861.210526] 00 [ 861.210527] 00 [ 861.210527] f3 [ 861.210527] 90 [ 861.210527] <41> [ 861.210528] 8b [ 861.210528] 40 [ 861.210528] 08 [ 861.210528] 85 [ 861.210529] c0 [ 861.210529] 74 [ 861.210529] f6 [ 861.210529] 4d [ 861.210530] 8b [ 861.210530] 08 [ 861.210530] 4d [ 861.210530] 85 [ 861.210531] c9 [ 861.210531] 74 [ 861.210531] 04 [ 861.210531] 41 [ 861.210532] 0f [ 861.210532] 18 [ 861.210532] 09 [ 861.210532] 8b [ 861.210532] [ 862.709036] intel_powerclamp [ 862.712839] coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt [ 862.791007] crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 862.829733] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 862.843997] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 862.856325] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 862.865160] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 862.876721] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 862.883132] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 862.891579] RDX: ffff9832bf4db8c0 RSI: 0000000000a10000 RDI: ffff97f2a83bcf40 [ 862.900027] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 862.908475] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 862.916922] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 862.925370] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 862.934884] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 862.941780] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 862.950227] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 862.958674] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 862.967121] Call Trace: [ 862.970335] [] queued_spin_lock_slowpath+0xb/0xf [ 862.977813] [] _raw_spin_lock+0x30/0x40 [ 862.984422] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 862.992101] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 863.000550] [] ? wake_up_atomic_t+0x30/0x30 [ 863.007547] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 863.014947] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 863.023219] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 863.031201] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 863.039860] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 863.047741] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 863.055048] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 863.063224] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 863.072158] [] ? __getnstimeofday64+0x3f/0xd0 [ 863.079365] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 863.088414] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 863.096476] [] ? __wake_up_common_lock+0x91/0xc0 [ 863.103953] [] ? sched_feat_set+0xf0/0xf0 [ 863.110769] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 863.118266] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 863.127007] [] kthread+0xd1/0xe0 [ 863.132935] [] ? insert_kthread_work+0x40/0x40 [ 863.140219] [] ret_from_fork_nospec_begin+0x21/0x21 [ 863.147988] [] ? insert_kthread_work+0x40/0x40 [ 863.155270] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 864.616084] Lustre: ls1-MDT0000: Export ffff97f26fe8ac00 already connecting from 192.168.128.8@o2ib42 [ 865.064651] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [mdt00_013:17594] [ 865.073683] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 865.212668] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_mgs_0014:10826] [ 865.153580] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi [ 865.212668] Modules linked in: [ 865.212669] osp(OE) [ 865.212670] mdd(OE) [ 865.212670] lod(OE) [ 865.212670] mdt(OE) [ 865.212671] lfsck(OE) [ 865.212671] mgs(OE) [ 865.212672] mgc(OE) [ 865.212672] osd_zfs(OE) [ 865.212673] lquota(OE) [ 865.212673] fid(OE) [ 865.212674] fld(OE) [ 865.212674] ptlrpc(OE) [ 865.212674] obdclass(OE) [ 865.212675] ko2iblnd(OE) [ 865.212675] lnet(OE) [ 865.212676] libcfs(OE) [ 865.212676] iTCO_wdt [ 865.212677] iTCO_vendor_support [ 865.212677] sb_edac [ 865.212677] intel_powerclamp [ 865.212678] coretemp [ 865.212678] intel_rapl [ 865.212679] iosf_mbi [ 865.212679] rpcrdma [ 865.212679] ib_iser [ 865.212680] kvm [ 865.212680] irqbypass [ 865.212680] joydev [ 865.212681] pcspkr [ 865.212681] lpc_ich [ 865.212682] i2c_i801 [ 865.212682] ioatdma [ 865.212682] ses [ 865.212683] enclosure [ 865.212683] sg [ 865.212683] ipmi_si [ 865.212684] ipmi_devintf [ 865.212684] ipmi_msghandler [ 865.212685] acpi_power_meter [ 865.212685] dm_round_robin [ 865.212686] acpi_cpufreq [ 865.212686] ib_ipoib [ 865.212686] rdma_ucm [ 865.212687] ib_umad [ 865.212687] sch_fq_codel [ 865.212688] iw_cxgb4 [ 865.212688] rdma_cm [ 865.212688] iw_cm [ 865.212689] ib_cm [ 865.212689] iw_cxgb3 [ 865.212690] zfs(POE) [ 865.212690] binfmt_misc [ 865.212690] zunicode(POE) [ 865.212691] zavl(POE) [ 865.212691] icp(POE) [ 865.212692] zcommon(POE) [ 865.212692] znvpair(POE) [ 865.212693] spl(OE) [ 865.212693] msr_safe(OE) [ 865.212694] ip_tables [ 865.212694] nfsv3 [ 865.212694] nfs_acl [ 865.212695] rpcsec_gss_krb5 [ 865.212695] auth_rpcgss [ 865.212696] nfsv4 [ 865.212696] dns_resolver [ 865.212697] nfs [ 865.212697] lockd [ 865.212697] grace [ 865.212698] fscache [ 865.212698] overlay(T) [ 865.212699] ext4 [ 865.212699] mbcache [ 865.212699] jbd2 [ 865.212700] dm_service_time [ 865.212700] sd_mod [ 865.212701] crc_t10dif [ 865.212701] crct10dif_generic [ 865.212701] mlx5_ib [ 865.212702] ib_uverbs [ 865.212702] ib_core [ 865.212703] be2iscsi [ 865.212703] bnx2i [ 865.212703] cnic [ 865.212704] uio [ 865.212704] cxgb4i [ 865.212705] cxgb4 [ 865.212705] cxgb3i [ 865.212705] cxgb3 [ 865.212706] mdio [ 865.212706] libcxgbi [ 865.212706] libcxgb [ 865.212707] qla4xxx [ 865.212707] iscsi_boot_sysfs [ 865.212708] mgag200 [ 865.212708] i2c_algo_bit [ 865.212709] 8021q [ 865.212709] drm_kms_helper [ 865.212709] garp [ 865.212710] syscopyarea [ 865.212710] mrp [ 865.212711] crct10dif_pclmul [ 865.212711] stp [ 865.212711] sysfillrect [ 865.212712] crct10dif_common [ 865.212712] sysimgblt [ 865.212713] crc32_pclmul [ 865.212713] llc [ 865.212713] fb_sys_fops [ 865.212714] crc32c_intel [ 865.212714] ttm [ 865.212715] ghash_clmulni_intel [ 865.212715] mlx5_core [ 865.212715] drm [ 865.212716] mxm_wmi [ 865.212716] ahci [ 865.212717] ixgbe(OE) [ 865.212717] aesni_intel [ 865.212717] mpt3sas [ 865.212718] mlxfw [ 865.212718] dm_multipath [ 865.212719] lrw [ 865.212719] devlink [ 865.212719] libahci [ 865.212720] gf128mul [ 865.212720] dca [ 865.212720] glue_helper [ 865.212721] ablk_helper [ 865.212721] raid_class [ 865.212722] ptp [ 865.212722] drm_panel_orientation_quirks [ 865.212723] libata [ 865.212723] cryptd [ 865.212723] scsi_transport_sas [ 865.212724] pps_core [ 865.212724] wmi [ 865.212725] sunrpc [ 865.212725] dm_mirror [ 865.212725] dm_region_hash [ 865.212726] dm_log [ 865.212726] dm_mod [ 865.212727] iscsi_tcp [ 865.212727] libiscsi_tcp [ 865.212727] libiscsi [ 865.212728] scsi_transport_iscsi [ 865.212728] fuse [ 865.212729] [ 865.212731] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 865.212732] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 865.212733] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 865.212734] RIP: 0010:[] [ 865.212739] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 865.212740] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 865.212741] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 865.212742] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 865.212743] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 865.212744] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 865.212745] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 865.212746] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 865.212747] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 865.212748] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 865.212749] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 865.212750] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 865.212751] Call Trace: [ 865.212756] [] queued_spin_lock_slowpath+0xb/0xf [ 865.212760] [] _raw_spin_lock+0x30/0x40 [ 865.212777] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 865.212782] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 865.212799] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 865.212802] [] ? wake_up_atomic_t+0x30/0x30 [ 865.212810] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 865.212859] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 865.212885] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 865.212887] [] ? check_preempt_curr+0x80/0xa0 [ 865.212910] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 865.212934] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 865.212973] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 865.213002] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 865.213004] [] ? __getnstimeofday64+0x3f/0xd0 [ 865.213030] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 865.213054] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 865.213056] [] ? __wake_up_common_lock+0x91/0xc0 [ 865.213057] [] ? sched_feat_set+0xf0/0xf0 [ 865.213082] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 865.213084] [] ? __switch_to+0xce/0x5a0 [ 865.213109] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 865.213111] [] kthread+0xd1/0xe0 [ 865.213113] [] ? insert_kthread_work+0x40/0x40 [ 865.213116] [] ret_from_fork_nospec_begin+0x21/0x21 [ 865.213118] [] ? insert_kthread_work+0x40/0x40 [ 865.213119] Code: [ 865.213119] 13 [ 865.213120] 48 [ 865.213120] c1 [ 865.213120] ea [ 865.213121] 0d [ 865.213121] 48 [ 865.213121] 98 [ 865.213122] 83 [ 865.213122] e2 [ 865.213123] 30 [ 865.213123] 48 [ 865.213124] 81 [ 865.213124] c2 [ 865.213124] c0 [ 865.213125] b8 [ 865.213125] 01 [ 865.213125] 00 [ 865.213126] 48 [ 865.213126] 03 [ 865.213126] 14 [ 865.213127] c5 [ 865.213127] e0 [ 865.213128] 17 [ 865.213128] d5 [ 865.213128] a0 [ 865.213129] 4c [ 865.213129] 89 [ 865.213129] 02 [ 865.213130] 41 [ 865.213130] 8b [ 865.213130] 40 [ 865.213131] 08 [ 865.213131] 85 [ 865.213131] c0 [ 865.213132] 75 [ 865.213132] 0f [ 865.213132] 0f [ 865.213133] 1f [ 865.213133] 44 [ 865.213134] 00 [ 865.213134] 00 [ 865.213134] f3 [ 865.213135] 90 [ 865.213135] <41> [ 865.213135] 8b [ 865.213136] 40 [ 865.213136] 08 [ 865.213136] 85 [ 865.213137] c0 [ 865.213137] 74 [ 865.213137] f6 [ 865.213138] 4d [ 865.213138] 8b [ 865.213138] 08 [ 865.213139] 4d [ 865.213139] 85 [ 865.213139] c9 [ 865.213140] 74 [ 865.213140] 04 [ 865.213140] 41 [ 865.213141] 0f [ 865.213141] 18 [ 865.213141] 09 [ 865.213142] 8b [ 865.213142] [ 865.224669] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 22s! [mdt01_014:17318] [ 865.224670] Modules linked in: [ 865.224671] osp(OE) [ 865.224671] mdd(OE) [ 865.224672] lod(OE) [ 865.224672] mdt(OE) [ 865.224672] lfsck(OE) [ 865.224673] mgs(OE) [ 865.224673] mgc(OE) [ 865.224673] osd_zfs(OE) [ 865.224673] lquota(OE) [ 865.224674] fid(OE) [ 865.224674] fld(OE) [ 865.224674] ptlrpc(OE) [ 865.224675] obdclass(OE) [ 865.224675] ko2iblnd(OE) [ 865.224675] lnet(OE) [ 865.224676] libcfs(OE) [ 865.224676] iTCO_wdt [ 865.224676] iTCO_vendor_support [ 865.224677] sb_edac [ 865.224677] intel_powerclamp [ 865.224677] coretemp [ 865.224677] intel_rapl [ 865.224678] iosf_mbi [ 865.224678] rpcrdma [ 865.224678] ib_iser [ 865.224679] kvm [ 865.224679] irqbypass [ 865.224679] joydev [ 865.224679] pcspkr [ 865.224680] lpc_ich [ 865.224680] i2c_i801 [ 865.224680] ioatdma [ 865.224681] ses [ 865.224681] enclosure [ 865.224681] sg [ 865.224681] ipmi_si [ 865.224682] ipmi_devintf [ 865.224682] ipmi_msghandler [ 865.224682] acpi_power_meter [ 865.224683] dm_round_robin [ 865.224683] acpi_cpufreq [ 865.224683] ib_ipoib [ 865.224683] rdma_ucm [ 865.224684] ib_umad [ 865.224684] sch_fq_codel [ 865.224684] iw_cxgb4 [ 865.224685] rdma_cm [ 865.224685] iw_cm [ 865.224685] ib_cm [ 865.224685] iw_cxgb3 [ 865.224686] zfs(POE) [ 865.224686] binfmt_misc [ 865.224686] zunicode(POE) [ 865.224687] zavl(POE) [ 865.224687] icp(POE) [ 865.224687] zcommon(POE) [ 865.224688] znvpair(POE) [ 865.224688] spl(OE) [ 865.224688] msr_safe(OE) [ 865.224689] ip_tables [ 865.224689] nfsv3 [ 865.224689] nfs_acl [ 865.224690] rpcsec_gss_krb5 [ 865.224690] auth_rpcgss [ 865.224690] nfsv4 [ 865.224690] dns_resolver [ 865.224691] nfs [ 865.224691] lockd [ 865.224691] grace [ 865.224692] fscache [ 865.224692] overlay(T) [ 865.224692] ext4 [ 865.224693] mbcache [ 865.224693] jbd2 [ 865.224693] dm_service_time [ 865.224694] sd_mod [ 865.224694] crc_t10dif [ 865.224695] crct10dif_generic [ 865.224695] mlx5_ib [ 865.224695] ib_uverbs [ 865.224695] ib_core [ 865.224696] be2iscsi [ 865.224696] bnx2i [ 865.224696] cnic [ 865.224697] uio [ 865.224697] cxgb4i [ 865.224697] cxgb4 [ 865.224698] cxgb3i [ 865.224698] cxgb3 [ 865.224698] mdio [ 865.224699] libcxgbi [ 865.224699] libcxgb [ 865.224699] qla4xxx [ 865.224700] iscsi_boot_sysfs [ 865.224700] mgag200 [ 865.224700] i2c_algo_bit [ 865.224701] 8021q [ 865.224701] drm_kms_helper [ 865.224701] garp [ 865.224701] syscopyarea [ 865.224702] mrp [ 865.224702] crct10dif_pclmul [ 865.224702] stp [ 865.224703] sysfillrect [ 865.224703] crct10dif_common [ 865.224703] sysimgblt [ 865.224704] crc32_pclmul [ 865.224704] llc [ 865.224704] fb_sys_fops [ 865.224704] crc32c_intel [ 865.224705] ttm [ 865.224705] ghash_clmulni_intel [ 865.224705] mlx5_core [ 865.224706] drm [ 865.224706] mxm_wmi [ 865.224706] ahci [ 865.224707] ixgbe(OE) [ 865.224707] aesni_intel [ 865.224707] mpt3sas [ 865.224708] mlxfw [ 865.224708] dm_multipath [ 865.224708] lrw [ 865.224708] devlink [ 865.224709] libahci [ 865.224709] gf128mul [ 865.224709] dca [ 865.224710] glue_helper [ 865.224710] ablk_helper [ 865.224711] raid_class [ 865.224711] ptp [ 865.224711] drm_panel_orientation_quirks [ 865.224711] libata [ 865.224712] cryptd [ 865.224712] scsi_transport_sas [ 865.224712] pps_core [ 865.224713] wmi [ 865.224713] sunrpc [ 865.224713] dm_mirror [ 865.224714] dm_region_hash [ 865.224714] dm_log [ 865.224714] dm_mod [ 865.224714] iscsi_tcp [ 865.224715] libiscsi_tcp [ 865.224715] libiscsi [ 865.224716] scsi_transport_iscsi [ 865.224716] fuse [ 865.224716] [ 865.224718] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 865.224718] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 865.224719] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 865.224720] RIP: 0010:[] [ 865.224723] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 865.224724] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 865.224724] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 865.224725] RDX: ffff9832bf21b8c0 RSI: 0000000000410000 RDI: ffff97f2a83bcf40 [ 865.224726] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 865.224726] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 865.224727] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 865.224728] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 865.224729] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 865.224729] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 865.224730] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 865.224731] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 865.224731] Call Trace: [ 865.224734] [] queued_spin_lock_slowpath+0xb/0xf [ 865.224736] [] _raw_spin_lock+0x30/0x40 [ 865.224741] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 865.224751] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 865.224753] [] ? wake_up_atomic_t+0x30/0x30 [ 865.224759] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 865.224784] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 865.224802] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 865.224821] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 865.224838] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 865.224855] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 865.224880] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 865.224901] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 865.224902] [] ? __getnstimeofday64+0x3f/0xd0 [ 865.224922] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 865.224940] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 865.224941] [] ? __wake_up_common_lock+0x91/0xc0 [ 865.224942] [] ? sched_feat_set+0xf0/0xf0 [ 865.224961] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 865.224979] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 865.224981] [] kthread+0xd1/0xe0 [ 865.224982] [] ? insert_kthread_work+0x40/0x40 [ 865.224983] [] ret_from_fork_nospec_begin+0x21/0x21 [ 865.224985] [] ? insert_kthread_work+0x40/0x40 [ 865.224985] Code: [ 865.224986] 0d [ 865.224986] 48 [ 865.224986] 98 [ 865.224986] 83 [ 865.224987] e2 [ 865.224987] 30 [ 865.224987] 48 [ 865.224987] 81 [ 865.224988] c2 [ 865.224988] c0 [ 865.224988] b8 [ 865.224988] 01 [ 865.224989] 00 [ 865.224989] 48 [ 865.224989] 03 [ 865.224989] 14 [ 865.224990] c5 [ 865.224990] e0 [ 865.224990] 17 [ 865.224990] d5 [ 865.224991] a0 [ 865.224991] 4c [ 865.224992] 89 [ 865.224992] 02 [ 865.224992] 41 [ 865.224993] 8b [ 865.224993] 40 [ 865.224993] 08 [ 865.224993] 85 [ 865.224994] c0 [ 865.224994] 75 [ 865.224994] 0f [ 865.224995] 0f [ 865.224995] 1f [ 865.224995] 44 [ 865.224995] 00 [ 865.224996] 00 [ 865.224996] f3 [ 865.224996] 90 [ 865.224996] 41 [ 865.224997] 8b [ 865.224997] 40 [ 865.224997] 08 [ 865.224998] <85> [ 865.224998] c0 [ 865.224998] 74 [ 865.224998] f6 [ 865.224999] 4d [ 865.224999] 8b [ 865.224999] 08 [ 865.224999] 4d [ 865.225000] 85 [ 865.225000] c9 [ 865.225000] 74 [ 865.225000] 04 [ 865.225001] 41 [ 865.225001] 0f [ 865.225001] 18 [ 865.225001] 09 [ 865.225001] 8b [ 865.225002] 17 [ 865.225002] 0f [ 865.225002] b7 [ 865.225002] c2 [ 865.225003] [ 866.222590] fuse [ 866.225228] [ 866.225433] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 866.239699] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 866.252024] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 866.260860] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 866.272421] RSP: 0018:ffff98321b877a50 EFLAGS: 00000246 [ 866.278833] RAX: 0000000000000000 RBX: ffff97f2bfa80000 RCX: 0000000000110000 [ 866.287281] RDX: ffff9832bf55b8c0 RSI: 0000000000e90000 RDI: ffff97f2a83bcf40 [ 866.295729] RBP: ffff98321b877a50 R08: ffff97f2bfa9b8c0 R09: 0000000000000000 [ 866.304175] R10: 0000000000000001 R11: 000000000000242e R12: ffff98321b877a18 [ 866.312624] R13: ffff97f2bfc9ad40 R14: ffffffffa00f0602 R15: ffff98321b877a28 [ 866.321072] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 866.330586] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 866.337482] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 866.345929] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 866.354377] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 866.362825] Call Trace: [ 866.366037] [] queued_spin_lock_slowpath+0xb/0xf [ 866.373516] [] _raw_spin_lock+0x30/0x40 [ 866.380131] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 866.387812] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 866.396261] [] ? wake_up_atomic_t+0x30/0x30 [ 866.403260] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 866.410664] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 866.418944] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 866.427513] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 866.435671] [] ? enqueue_task_fair+0x208/0x6c0 [ 866.442958] [] ? sched_clock_cpu+0xa5/0xe0 [ 866.449854] [] ? check_preempt_curr+0x80/0xa0 [ 866.457041] [] ? ttwu_do_wakeup+0x19/0x100 [ 866.463967] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 866.472145] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 866.481080] [] ? __getnstimeofday64+0x3f/0xd0 [ 866.488292] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 866.497346] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 866.505407] [] ? __wake_up_common_lock+0x91/0xc0 [ 866.512885] [] ? sched_feat_set+0xf0/0xf0 [ 866.519709] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 866.527187] [] ? __switch_to+0xce/0x5a0 [ 866.533816] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 866.542556] [] kthread+0xd1/0xe0 [ 866.548485] [] ? insert_kthread_work+0x40/0x40 [ 866.555769] [] ret_from_fork_nospec_begin+0x21/0x21 [ 866.563539] [] ? insert_kthread_work+0x40/0x40 [ 866.570822] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 868.183010] INFO: rcu_sched self-detected stall on CPU [ 868.184008] INFO: rcu_sched self-detected stall on CPU [ 868.184010] INFO: rcu_sched self-detected stall on CPU [ 868.184011] INFO: rcu_sched self-detected stall on CPU [ 868.184012] INFO: rcu_sched self-detected stall on CPU [ 868.184013] INFO: rcu_sched self-detected stall on CPU [ 868.184014] INFO: rcu_sched self-detected stall on CPU [ 868.184015] INFO: rcu_sched self-detected stall on CPU [ 868.184016] INFO: rcu_sched self-detected stall on CPU [ 868.184017] { [ 868.184018] { [ 868.184019] { [ 868.184019] { [ 868.184020] { [ 868.184021] { [ 868.184021] { [ 868.184022] { [ 868.184023] 29 [ 868.184024] 23 [ 868.184025] 14 [ 868.184025] 30 [ 868.184027] 2 [ 868.184027] 15 [ 868.184028] 8 [ 868.184029] 20 [ 868.184030] } [ 868.184031] } [ 868.184031] } [ 868.184032] } [ 868.184032] } [ 868.184033] } [ 868.184033] } [ 868.184034] } [ 868.184036] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184037] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184039] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184040] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184041] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184042] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184044] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184045] (t=60000 jiffies g=29074 c=29073 q=168213) [ 868.184046] Task dump for CPU 2: [ 868.184047] mdt00_013 R [ 868.184047] running task [ 868.184048] 0 17594 2 0x00000088 [ 868.184049] Call Trace: [ 868.184051] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.184053] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.184055] [] ? _raw_spin_lock+0x30/0x40 [ 868.184061] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.184067] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.184075] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.184077] [] ? wake_up_atomic_t+0x30/0x30 [ 868.184084] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.184107] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.184128] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.184150] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.184153] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.184155] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.184156] [] ? check_preempt_curr+0x80/0xa0 [ 868.184157] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.184186] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.184212] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.184213] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.184237] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.184260] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.184262] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.184263] [] ? sched_feat_set+0xf0/0xf0 [ 868.184286] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.184289] [] ? __switch_to+0xce/0x5a0 [ 868.184311] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.184313] [] ? kthread+0xd1/0xe0 [ 868.184315] [] ? insert_kthread_work+0x40/0x40 [ 868.184316] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.184318] [] ? insert_kthread_work+0x40/0x40 [ 868.184318] Task dump for CPU 8: [ 868.184319] mdt01_022 R [ 868.184319] running task [ 868.184320] 0 17601 2 0x00000088 [ 868.184320] Call Trace: [ 868.184322] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.184324] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.184326] [] ? _raw_spin_lock+0x30/0x40 [ 868.184332] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.184340] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.184342] [] ? wake_up_atomic_t+0x30/0x30 [ 868.184349] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.184371] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.184392] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.184415] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.184438] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.184459] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.184487] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.184513] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.184514] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.184538] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.184560] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.184562] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.184563] [] ? sched_feat_set+0xf0/0xf0 [ 868.184586] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.184608] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.184610] [] ? kthread+0xd1/0xe0 [ 868.184612] [] ? insert_kthread_work+0x40/0x40 [ 868.184614] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.184615] [] ? insert_kthread_work+0x40/0x40 [ 868.184616] Task dump for CPU 13: [ 868.184616] mdt01_003 R [ 868.184617] running task [ 868.184617] 0 10825 2 0x00000088 [ 868.184618] Call Trace: [ 868.184619] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.184622] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.184623] [] ? _raw_spin_lock+0x30/0x40 [ 868.184629] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.184637] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.184639] [] ? wake_up_atomic_t+0x30/0x30 [ 868.184646] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.184668] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.184687] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.184689] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.184690] [] ? check_preempt_curr+0x80/0xa0 [ 868.184692] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.184720] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.184745] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.184747] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.184770] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.184792] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.184794] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.184795] [] ? sched_feat_set+0xf0/0xf0 [ 868.184817] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.184840] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.184842] [] ? kthread+0xd1/0xe0 [ 868.184843] [] ? insert_kthread_work+0x40/0x40 [ 868.184845] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.184847] [] ? insert_kthread_work+0x40/0x40 [ 868.184847] Task dump for CPU 14: [ 868.184848] mdt01_016 R [ 868.184848] running task [ 868.184849] 0 17554 2 0x00000088 [ 868.184849] Call Trace: [ 868.184851] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.184852] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.184854] [] ? _raw_spin_lock+0x30/0x40 [ 868.184860] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.184867] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.184869] [] ? wake_up_atomic_t+0x30/0x30 [ 868.184876] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.184898] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.184920] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.184922] [] ? check_preempt_curr+0x80/0xa0 [ 868.184944] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.184965] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.184993] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.185018] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.185019] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.185043] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.185066] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.185067] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.185068] [] ? sched_feat_set+0xf0/0xf0 [ 868.185091] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.185113] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.185115] [] ? kthread+0xd1/0xe0 [ 868.185116] [] ? insert_kthread_work+0x40/0x40 [ 868.185118] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.185119] [] ? insert_kthread_work+0x40/0x40 [ 868.185120] Task dump for CPU 15: [ 868.185121] mdt01_007 R [ 868.185121] running task [ 868.185122] 0 17310 2 0x00000088 [ 868.185122] Call Trace: [ 868.185124] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.185126] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.185127] [] ? _raw_spin_lock+0x30/0x40 [ 868.185133] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.185141] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.185143] [] ? wake_up_atomic_t+0x30/0x30 [ 868.185150] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.185172] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.185193] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.185195] [] ? check_preempt_curr+0x80/0xa0 [ 868.185217] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.185238] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.185265] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.185291] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.185292] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.185316] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.185338] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.185340] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.185341] [] ? sched_feat_set+0xf0/0xf0 [ 868.185364] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.185387] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.185388] [] ? kthread+0xd1/0xe0 [ 868.185390] [] ? insert_kthread_work+0x40/0x40 [ 868.185392] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.185393] [] ? insert_kthread_work+0x40/0x40 [ 868.185394] Task dump for CPU 20: [ 868.185395] Task dump for CPU 2: [ 868.185396] ll_mgs_0013 R [ 868.185397] mdt00_013 R [ 868.185398] running task [ 868.185399] running task [ 868.185400] 0 10822 2 0x00000088 [ 868.185401] 0 17594 2 0x00000088 [ 868.185402] Call Trace: [ 868.185403] Call Trace: [ 868.185406] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.185408] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.185411] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.185413] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.185415] [] ? _raw_spin_lock+0x30/0x40 [ 868.185417] [] ? _raw_spin_lock+0x30/0x40 [ 868.185423] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.185430] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.185438] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.185442] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.185445] [] ? wake_up_atomic_t+0x30/0x30 [ 868.185453] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.185461] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.185463] [] ? wake_up_atomic_t+0x30/0x30 [ 868.185485] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.185491] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.185511] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.185540] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.185542] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.185559] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.185561] [] ? check_preempt_curr+0x80/0xa0 [ 868.185579] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.185581] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.185583] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.185611] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.185613] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.185640] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.185641] [] ? check_preempt_curr+0x80/0xa0 [ 868.185643] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.185645] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.185669] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.185694] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.185717] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.185739] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.185741] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.185743] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.185745] [] ? sched_feat_set+0xf0/0xf0 [ 868.185770] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.185793] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.185819] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.185821] [] ? __switch_to+0xce/0x5a0 [ 868.185823] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.185845] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.185847] [] ? sched_feat_set+0xf0/0xf0 [ 868.185849] [] ? kthread+0xd1/0xe0 [ 868.185874] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.185876] [] ? insert_kthread_work+0x40/0x40 [ 868.185878] [] ? __switch_to+0xce/0x5a0 [ 868.185880] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.185906] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.185908] [] ? insert_kthread_work+0x40/0x40 [ 868.185910] [] ? kthread+0xd1/0xe0 [ 868.185911] Task dump for CPU 23: [ 868.185913] [] ? insert_kthread_work+0x40/0x40 [ 868.185914] ll_mgs_0014 R [ 868.185916] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.185917] running task [ 868.185918] [] ? insert_kthread_work+0x40/0x40 [ 868.185920] 0 10826 2 0x00000088 [ 868.185920] Task dump for CPU 8: [ 868.185921] Call Trace: [ 868.185922] mdt01_022 R [ 868.185925] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.185926] running task [ 868.185928] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.185929] 0 17601 2 0x00000088 [ 868.185931] [] ? _raw_spin_lock+0x30/0x40 [ 868.185932] Call Trace: [ 868.185938] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.185940] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.185948] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.185950] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.185952] [] ? wake_up_atomic_t+0x30/0x30 [ 868.185954] [] ? _raw_spin_lock+0x30/0x40 [ 868.185962] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.185967] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.185989] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.185996] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.186019] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.186021] [] ? wake_up_atomic_t+0x30/0x30 [ 868.186023] [] ? check_preempt_curr+0x80/0xa0 [ 868.186029] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.186051] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.186070] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.186092] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.186110] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.186138] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.186157] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.186183] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.186201] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.186203] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.186220] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.186244] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.186267] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.186290] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.186311] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.186313] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.186315] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.186317] [] ? sched_feat_set+0xf0/0xf0 [ 868.186337] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.186360] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.186379] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.186381] [] ? __switch_to+0xce/0x5a0 [ 868.186383] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.186405] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.186407] [] ? sched_feat_set+0xf0/0xf0 [ 868.186409] [] ? kthread+0xd1/0xe0 [ 868.186427] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.186429] [] ? insert_kthread_work+0x40/0x40 [ 868.186447] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.186449] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.186450] [] ? kthread+0xd1/0xe0 [ 868.186453] [] ? insert_kthread_work+0x40/0x40 [ 868.186455] [] ? insert_kthread_work+0x40/0x40 [ 868.186456] Task dump for CPU 29: [ 868.186458] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.186459] mdt01_017 R [ 868.186460] [] ? insert_kthread_work+0x40/0x40 [ 868.186461] running task [ 868.186462] Task dump for CPU 13: [ 868.186464] 0 17555 2 0x00000088 [ 868.186465] mdt01_003 R [ 868.186466] Call Trace: [ 868.186466] running task [ 868.186467] 0 10825 2 0x00000088 [ 868.186468] [ 868.186469] Call Trace: [ 868.186472] [] sched_show_task+0xbf/0x120 [ 868.186474] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.186476] [] dump_cpu_task+0x39/0x70 [ 868.186478] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.186483] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.186484] [] ? _raw_spin_lock+0x30/0x40 [ 868.186488] [] rcu_check_callbacks+0x482/0x770 [ 868.186493] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.186497] [] update_process_times+0x46/0x80 [ 868.186504] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.186507] [] tick_sched_handle+0x30/0x70 [ 868.186509] [] ? wake_up_atomic_t+0x30/0x30 [ 868.186511] [] tick_sched_timer+0x39/0x80 [ 868.186517] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.186519] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.186537] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.186539] [] ? tick_sched_do_timer+0x50/0x50 [ 868.186555] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.186557] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.186559] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.186562] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.186564] [] ? check_preempt_curr+0x80/0xa0 [ 868.186567] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.186568] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.186571] [] apic_timer_interrupt+0x16a/0x170 [ 868.186593] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.186594] [ 868.186615] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.186617] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.186619] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.186621] [] queued_spin_lock_slowpath+0xb/0xf [ 868.186640] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.186642] [] _raw_spin_lock+0x30/0x40 [ 868.186660] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.186666] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.186668] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.186676] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.186677] [] ? sched_feat_set+0xf0/0xf0 [ 868.186680] [] ? wake_up_atomic_t+0x30/0x30 [ 868.186698] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.186705] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.186723] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.186745] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.186747] [] ? kthread+0xd1/0xe0 [ 868.186770] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.186772] [] ? insert_kthread_work+0x40/0x40 [ 868.186774] [] ? check_preempt_curr+0x80/0xa0 [ 868.186776] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.186797] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.186799] [] ? insert_kthread_work+0x40/0x40 [ 868.186820] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.186821] Task dump for CPU 14: [ 868.186850] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.186851] mdt01_016 R [ 868.186877] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.186878] running task [ 868.186880] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.186881] 0 17554 2 0x00000088 [ 868.186905] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.186906] Call Trace: [ 868.186929] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.186931] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.186933] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.186935] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.186937] [] ? sched_feat_set+0xf0/0xf0 [ 868.186940] [] ? _raw_spin_lock+0x30/0x40 [ 868.186963] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.186968] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.186990] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.186997] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.186999] [] kthread+0xd1/0xe0 [ 868.187001] [] ? wake_up_atomic_t+0x30/0x30 [ 868.187003] [] ? insert_kthread_work+0x40/0x40 [ 868.187010] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.187012] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.187029] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.187032] [] ? insert_kthread_work+0x40/0x40 [ 868.187049] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.187050] Task dump for CPU 30: [ 868.187052] [] ? check_preempt_curr+0x80/0xa0 [ 868.187053] mdt01_018 R [ 868.187071] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.187071] running task [ 868.187088] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.187089] 0 17592 2 0x00000088 [ 868.187112] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.187113] Call Trace: [ 868.187134] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.187136] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.187138] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.187140] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.187159] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.187161] [] ? _raw_spin_lock+0x30/0x40 [ 868.187180] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.187186] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.187188] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.187196] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.187197] [] ? sched_feat_set+0xf0/0xf0 [ 868.187199] [] ? wake_up_atomic_t+0x30/0x30 [ 868.187218] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.187225] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.187243] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.187265] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.187267] [] ? kthread+0xd1/0xe0 [ 868.187287] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.187289] [] ? insert_kthread_work+0x40/0x40 [ 868.187292] [] ? check_preempt_curr+0x80/0xa0 [ 868.187293] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.187295] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.187297] [] ? insert_kthread_work+0x40/0x40 [ 868.187325] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.187326] Task dump for CPU 15: [ 868.187353] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.187354] mdt01_007 R [ 868.187356] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.187356] running task [ 868.187380] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.187382] 0 17310 2 0x00000088 [ 868.187405] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.187405] Call Trace: [ 868.187407] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.187409] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.187411] [] ? sched_feat_set+0xf0/0xf0 [ 868.187413] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.187436] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.187438] [] ? _raw_spin_lock+0x30/0x40 [ 868.187460] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.187465] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.187467] [] ? kthread+0xd1/0xe0 [ 868.187474] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.187476] [] ? insert_kthread_work+0x40/0x40 [ 868.187478] [] ? wake_up_atomic_t+0x30/0x30 [ 868.187481] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.187486] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.187489] [] ? insert_kthread_work+0x40/0x40 [ 868.187506] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.187524] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.187525] [] ? check_preempt_curr+0x80/0xa0 [ 868.187542] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.187559] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.187581] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.187602] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.187603] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.187622] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.187640] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.187641] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.187642] [] ? sched_feat_set+0xf0/0xf0 [ 868.187660] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.187678] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.187679] [] ? kthread+0xd1/0xe0 [ 868.187681] [] ? insert_kthread_work+0x40/0x40 [ 868.187682] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.187683] [] ? insert_kthread_work+0x40/0x40 [ 868.187684] Task dump for CPU 20: [ 868.187685] Task dump for CPU 2: [ 868.187686] ll_mgs_0013 R [ 868.187687] mdt00_013 R [ 868.187688] running task [ 868.187689] running task [ 868.187690] 0 10822 2 0x00000088 [ 868.187692] 0 17594 2 0x00000088 [ 868.187692] Call Trace: [ 868.187693] Call Trace: [ 868.187695] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.187698] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.187701] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.187703] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.187705] [] ? _raw_spin_lock+0x30/0x40 [ 868.187707] [] ? _raw_spin_lock+0x30/0x40 [ 868.187712] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.187719] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.187725] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.187731] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.187733] [] ? wake_up_atomic_t+0x30/0x30 [ 868.187742] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.187748] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.187750] [] ? wake_up_atomic_t+0x30/0x30 [ 868.187768] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.187776] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.187791] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.187817] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.187819] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.187842] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.187844] [] ? check_preempt_curr+0x80/0xa0 [ 868.187870] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.187872] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.187874] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.187897] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.187899] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.187920] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.187922] [] ? check_preempt_curr+0x80/0xa0 [ 868.187924] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.187926] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.187946] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.187977] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.187996] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.188025] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.188027] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.188029] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.188030] [] ? sched_feat_set+0xf0/0xf0 [ 868.188058] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.188076] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.188102] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.188104] [] ? __switch_to+0xce/0x5a0 [ 868.188106] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.188124] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.188126] [] ? sched_feat_set+0xf0/0xf0 [ 868.188128] [] ? kthread+0xd1/0xe0 [ 868.188153] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.188154] [] ? insert_kthread_work+0x40/0x40 [ 868.188157] [] ? __switch_to+0xce/0x5a0 [ 868.188159] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.188184] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.188186] [] ? insert_kthread_work+0x40/0x40 [ 868.188188] [] ? kthread+0xd1/0xe0 [ 868.188189] Task dump for CPU 23: [ 868.188192] [] ? insert_kthread_work+0x40/0x40 [ 868.188193] ll_mgs_0014 R [ 868.188195] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.188195] running task [ 868.188198] [] ? insert_kthread_work+0x40/0x40 [ 868.188199] 0 10826 2 0x00000088 [ 868.188200] Task dump for CPU 8: [ 868.188201] Call Trace: [ 868.188202] mdt01_022 R [ 868.188203] [ 868.188204] running task [ 868.188206] [] sched_show_task+0xbf/0x120 [ 868.188207] 0 17601 2 0x00000088 [ 868.188210] [] dump_cpu_task+0x39/0x70 [ 868.188211] Call Trace: [ 868.188214] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.188217] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.188219] [] rcu_check_callbacks+0x482/0x770 [ 868.188222] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.188225] [] update_process_times+0x46/0x80 [ 868.188227] [] ? _raw_spin_lock+0x30/0x40 [ 868.188230] [] tick_sched_handle+0x30/0x70 [ 868.188237] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.188239] [] tick_sched_timer+0x39/0x80 [ 868.188247] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.188249] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.188251] [] ? wake_up_atomic_t+0x30/0x30 [ 868.188253] [] ? tick_sched_do_timer+0x50/0x50 [ 868.188261] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.188263] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.188287] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.188290] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.188315] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.188317] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.188343] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.188345] [] apic_timer_interrupt+0x16a/0x170 [ 868.188370] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.188371] [ 868.188394] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.188396] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.188427] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.188429] [] queued_spin_lock_slowpath+0xb/0xf [ 868.188458] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.188460] [] _raw_spin_lock+0x30/0x40 [ 868.188462] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.188467] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.188494] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.188500] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.188526] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.188528] [] ? wake_up_atomic_t+0x30/0x30 [ 868.188530] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.188536] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.188538] [] ? sched_feat_set+0xf0/0xf0 [ 868.188556] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.188582] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.188600] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.188625] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.188627] [] ? check_preempt_curr+0x80/0xa0 [ 868.188630] [] ? kthread+0xd1/0xe0 [ 868.188646] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.188649] [] ? insert_kthread_work+0x40/0x40 [ 868.188666] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.188668] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.188690] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.188693] [] ? insert_kthread_work+0x40/0x40 [ 868.188714] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.188715] Task dump for CPU 13: [ 868.188717] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.188718] mdt01_003 R [ 868.188737] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.188738] running task [ 868.188757] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.188758] 0 10825 2 0x00000088 [ 868.188760] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.188761] Call Trace: [ 868.188762] [] ? sched_feat_set+0xf0/0xf0 [ 868.188765] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.188783] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.188785] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.188787] [] ? __switch_to+0xce/0x5a0 [ 868.188789] [] ? _raw_spin_lock+0x30/0x40 [ 868.188807] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.188814] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.188815] [] kthread+0xd1/0xe0 [ 868.188824] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.188826] [] ? insert_kthread_work+0x40/0x40 [ 868.188829] [] ? wake_up_atomic_t+0x30/0x30 [ 868.188831] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.188839] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.188841] [] ? insert_kthread_work+0x40/0x40 [ 868.188866] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.188867] Task dump for CPU 29: [ 868.188889] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.188890] mdt01_017 R [ 868.188893] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.188894] running task [ 868.188896] [] ? check_preempt_curr+0x80/0xa0 [ 868.188898] 0 17555 2 0x00000088 [ 868.188900] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.188901] Call Trace: [ 868.188932] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.188934] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.188963] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.188965] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.188967] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.188969] [] ? _raw_spin_lock+0x30/0x40 [ 868.188995] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.189001] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.189026] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.189032] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.189035] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.189037] [] ? wake_up_atomic_t+0x30/0x30 [ 868.189040] [] ? sched_feat_set+0xf0/0xf0 [ 868.189046] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.189071] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.189089] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.189114] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.189132] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.189135] [] ? kthread+0xd1/0xe0 [ 868.189137] [] ? check_preempt_curr+0x80/0xa0 [ 868.189139] [] ? insert_kthread_work+0x40/0x40 [ 868.189156] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.189159] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.189175] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.189178] [] ? insert_kthread_work+0x40/0x40 [ 868.189200] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.189201] Task dump for CPU 14: [ 868.189222] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.189224] mdt01_016 R [ 868.189225] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.189226] running task [ 868.189246] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.189247] 0 17554 2 0x00000088 [ 868.189266] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.189267] Call Trace: [ 868.189268] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.189271] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.189272] [] ? sched_feat_set+0xf0/0xf0 [ 868.189275] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.189293] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.189295] [] ? _raw_spin_lock+0x30/0x40 [ 868.189313] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.189320] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.189322] [] ? kthread+0xd1/0xe0 [ 868.189331] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.189333] [] ? insert_kthread_work+0x40/0x40 [ 868.189335] [] ? wake_up_atomic_t+0x30/0x30 [ 868.189337] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.189345] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.189347] [] ? insert_kthread_work+0x40/0x40 [ 868.189371] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.189372] Task dump for CPU 30: [ 868.189397] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.189398] mdt01_018 R [ 868.189401] [] ? check_preempt_curr+0x80/0xa0 [ 868.189401] running task [ 868.189402] 0 17592 2 0x00000088 [ 868.189426] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.189427] Call Trace: [ 868.189451] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.189453] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.189484] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.189486] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.189515] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.189516] [] ? _raw_spin_lock+0x30/0x40 [ 868.189519] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.189524] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.189550] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.189556] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.189583] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.189585] [] ? wake_up_atomic_t+0x30/0x30 [ 868.189587] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.189593] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.189595] [] ? sched_feat_set+0xf0/0xf0 [ 868.189613] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.189638] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.189660] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.189685] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.189687] [] ? check_preempt_curr+0x80/0xa0 [ 868.189690] [] ? kthread+0xd1/0xe0 [ 868.189692] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.189694] [] ? insert_kthread_work+0x40/0x40 [ 868.189721] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.189724] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.189750] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.189753] [] ? insert_kthread_work+0x40/0x40 [ 868.189754] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.189755] Task dump for CPU 15: [ 868.189780] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.189781] mdt01_007 R [ 868.189807] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.189807] running task [ 868.189809] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.189810] 0 17310 2 0x00000088 [ 868.189812] [] ? sched_feat_set+0xf0/0xf0 [ 868.189813] Call Trace: [ 868.189838] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.189841] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.189866] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.189869] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.189871] [] ? kthread+0xd1/0xe0 [ 868.189873] [] ? _raw_spin_lock+0x30/0x40 [ 868.189875] [] ? insert_kthread_work+0x40/0x40 [ 868.189882] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.189884] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.189892] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.189894] [] ? insert_kthread_work+0x40/0x40 [ 868.189896] [] ? wake_up_atomic_t+0x30/0x30 [ 868.189904] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.189929] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.189954] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.189955] [] ? check_preempt_curr+0x80/0xa0 [ 868.189980] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.190003] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.190034] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.190062] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.190064] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.190091] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.190116] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.190117] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.190119] [] ? sched_feat_set+0xf0/0xf0 [ 868.190143] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.190168] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.190170] [] ? kthread+0xd1/0xe0 [ 868.190172] [] ? insert_kthread_work+0x40/0x40 [ 868.190174] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.190175] [] ? insert_kthread_work+0x40/0x40 [ 868.190176] Task dump for CPU 20: [ 868.190177] Task dump for CPU 2: [ 868.190177] ll_mgs_0013 R [ 868.190178] mdt00_013 R [ 868.190178] running task [ 868.190178] running task [ 868.190179] 0 10822 2 0x00000088 [ 868.190180] 0 17594 2 0x00000088 [ 868.190180] Call Trace: [ 868.190181] Call Trace: [ 868.190182] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.190184] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.190186] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.190188] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.190190] [] ? _raw_spin_lock+0x30/0x40 [ 868.190191] [] ? _raw_spin_lock+0x30/0x40 [ 868.190197] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.190202] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.190210] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.190216] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.190218] [] ? wake_up_atomic_t+0x30/0x30 [ 868.190225] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.190232] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.190234] [] ? wake_up_atomic_t+0x30/0x30 [ 868.190258] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.190265] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.190288] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.190313] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.190315] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.190336] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.190338] [] ? check_preempt_curr+0x80/0xa0 [ 868.190362] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.190364] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.190366] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.190396] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.190399] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.190426] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.190428] [] ? check_preempt_curr+0x80/0xa0 [ 868.190430] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.190431] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.190457] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.190487] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.190513] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.190541] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.190542] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.190544] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.190545] [] ? sched_feat_set+0xf0/0xf0 [ 868.190571] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.190597] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.190621] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.190623] [] ? __switch_to+0xce/0x5a0 [ 868.190624] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.190648] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.190650] [] ? sched_feat_set+0xf0/0xf0 [ 868.190652] [] ? kthread+0xd1/0xe0 [ 868.190675] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.190677] [] ? insert_kthread_work+0x40/0x40 [ 868.190679] [] ? __switch_to+0xce/0x5a0 [ 868.190681] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.190704] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.190706] [] ? insert_kthread_work+0x40/0x40 [ 868.190708] [] ? kthread+0xd1/0xe0 [ 868.190709] Task dump for CPU 23: [ 868.190710] [] ? insert_kthread_work+0x40/0x40 [ 868.190711] ll_mgs_0014 R [ 868.190712] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.190713] running task [ 868.190714] [] ? insert_kthread_work+0x40/0x40 [ 868.190715] 0 10826 2 0x00000088 [ 868.190716] Task dump for CPU 8: [ 868.190716] Call Trace: [ 868.190716] mdt01_022 R [ 868.190718] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.190718] running task [ 868.190720] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.190721] 0 17601 2 0x00000088 [ 868.190723] [] ? _raw_spin_lock+0x30/0x40 [ 868.190723] Call Trace: [ 868.190725] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.190731] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.190733] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.190741] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.190742] [] ? _raw_spin_lock+0x30/0x40 [ 868.190744] [] ? wake_up_atomic_t+0x30/0x30 [ 868.190749] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.190757] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.190764] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.190788] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.190790] [] ? wake_up_atomic_t+0x30/0x30 [ 868.190815] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.190823] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.190825] [] ? check_preempt_curr+0x80/0xa0 [ 868.190848] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.190872] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.190896] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.190919] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.190944] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.190975] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.190999] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.191027] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.191052] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.191054] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.191083] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.191109] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.191138] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.191164] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.191166] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.191167] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.191192] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.191194] [] ? sched_feat_set+0xf0/0xf0 [ 868.191218] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.191242] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.191244] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.191245] [] ? __switch_to+0xce/0x5a0 [ 868.191247] [] ? sched_feat_set+0xf0/0xf0 [ 868.191271] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.191295] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.191297] [] ? kthread+0xd1/0xe0 [ 868.191321] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.191323] [] ? insert_kthread_work+0x40/0x40 [ 868.191324] [] ? kthread+0xd1/0xe0 [ 868.191326] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.191327] [] ? insert_kthread_work+0x40/0x40 [ 868.191329] [] ? insert_kthread_work+0x40/0x40 [ 868.191331] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.191332] Task dump for CPU 29: [ 868.191333] [] ? insert_kthread_work+0x40/0x40 [ 868.191334] mdt01_017 R [ 868.191334] Task dump for CPU 13: [ 868.191335] running task [ 868.191335] mdt01_003 R [ 868.191336] 0 17555 2 0x00000088 [ 868.191336] running task [ 868.191336] Call Trace: [ 868.191337] 0 10825 2 0x00000088 [ 868.191339] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.191339] Call Trace: [ 868.191341] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.191343] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.191344] [] ? _raw_spin_lock+0x30/0x40 [ 868.191346] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.191352] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.191353] [] ? _raw_spin_lock+0x30/0x40 [ 868.191361] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.191366] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.191368] [] ? wake_up_atomic_t+0x30/0x30 [ 868.191375] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.191383] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.191384] [] ? wake_up_atomic_t+0x30/0x30 [ 868.191409] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.191416] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.191441] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.191465] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.191467] [] ? check_preempt_curr+0x80/0xa0 [ 868.191489] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.191514] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.191516] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.191540] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.191542] [] ? check_preempt_curr+0x80/0xa0 [ 868.191572] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.191574] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.191602] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.191631] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.191633] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.191660] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.191687] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.191689] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.191714] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.191738] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.191740] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.191764] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.191766] [] ? sched_feat_set+0xf0/0xf0 [ 868.191767] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.191791] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.191793] [] ? sched_feat_set+0xf0/0xf0 [ 868.191817] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.191841] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.191843] [] ? kthread+0xd1/0xe0 [ 868.191867] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.191869] [] ? insert_kthread_work+0x40/0x40 [ 868.191870] [] ? kthread+0xd1/0xe0 [ 868.191872] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.191874] [] ? insert_kthread_work+0x40/0x40 [ 868.191875] [] ? insert_kthread_work+0x40/0x40 [ 868.191877] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.191877] Task dump for CPU 30: [ 868.191879] [] ? insert_kthread_work+0x40/0x40 [ 868.191879] mdt01_018 R [ 868.191880] Task dump for CPU 14: [ 868.191880] running task [ 868.191881] mdt01_016 R [ 868.191881] 0 17592 2 0x00000088 [ 868.191882] running task [ 868.191882] Call Trace: [ 868.191883] 0 17554 2 0x00000088 [ 868.191883] [ 868.191883] Call Trace: [ 868.191885] [] sched_show_task+0xbf/0x120 [ 868.191886] [ 868.191887] [] dump_cpu_task+0x39/0x70 [ 868.191889] [] sched_show_task+0xbf/0x120 [ 868.191891] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.191892] [] dump_cpu_task+0x39/0x70 [ 868.191894] [] rcu_check_callbacks+0x482/0x770 [ 868.191896] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.191897] [] update_process_times+0x46/0x80 [ 868.191899] [] rcu_check_callbacks+0x482/0x770 [ 868.191901] [] tick_sched_handle+0x30/0x70 [ 868.191903] [] update_process_times+0x46/0x80 [ 868.191904] [] tick_sched_timer+0x39/0x80 [ 868.191906] [] tick_sched_handle+0x30/0x70 [ 868.191907] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.191909] [] tick_sched_timer+0x39/0x80 [ 868.191911] [] ? tick_sched_do_timer+0x50/0x50 [ 868.191912] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.191913] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.191915] [] ? tick_sched_do_timer+0x50/0x50 [ 868.191916] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.191918] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.191920] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.191921] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.191922] [] apic_timer_interrupt+0x16a/0x170 [ 868.191924] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.191924] [ 868.191926] [] apic_timer_interrupt+0x16a/0x170 [ 868.191927] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.191928] [ 868.191930] [] queued_spin_lock_slowpath+0xb/0xf [ 868.191931] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.191933] [] _raw_spin_lock+0x30/0x40 [ 868.191935] [] queued_spin_lock_slowpath+0xb/0xf [ 868.191941] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.191942] [] _raw_spin_lock+0x30/0x40 [ 868.191950] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.191956] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.191958] [] ? wake_up_atomic_t+0x30/0x30 [ 868.191965] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.191973] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.191975] [] ? wake_up_atomic_t+0x30/0x30 [ 868.191999] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.192007] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.192029] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.192053] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.192055] [] ? check_preempt_curr+0x80/0xa0 [ 868.192080] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.192082] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.192083] [] ? check_preempt_curr+0x80/0xa0 [ 868.192113] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.192138] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.192166] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.192191] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.192193] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.192222] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.192248] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.192276] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.192302] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.192303] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.192305] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.192329] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.192331] [] ? sched_feat_set+0xf0/0xf0 [ 868.192355] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.192380] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.192382] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.192405] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.192407] [] ? sched_feat_set+0xf0/0xf0 [ 868.192409] [] kthread+0xd1/0xe0 [ 868.192433] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.192435] [] ? insert_kthread_work+0x40/0x40 [ 868.192458] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.192460] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.192462] [] kthread+0xd1/0xe0 [ 868.192463] [] ? insert_kthread_work+0x40/0x40 [ 868.192465] [] ? insert_kthread_work+0x40/0x40 [ 868.192467] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.192468] [] ? insert_kthread_work+0x40/0x40 [ 868.192469] Task dump for CPU 15: [ 868.192470] mdt01_007 R [ 868.192470] running task [ 868.192471] 0 17310 2 0x00000088 [ 868.192471] Call Trace: [ 868.192473] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.192475] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.192477] [] ? _raw_spin_lock+0x30/0x40 [ 868.192484] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.192492] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.192494] [] ? wake_up_atomic_t+0x30/0x30 [ 868.192501] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.192524] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.192548] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.192550] [] ? check_preempt_curr+0x80/0xa0 [ 868.192575] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.192599] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.192629] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.192657] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.192659] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.192684] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.192709] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.192710] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.192712] [] ? sched_feat_set+0xf0/0xf0 [ 868.192736] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.192760] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.192762] [] ? kthread+0xd1/0xe0 [ 868.192764] [] ? insert_kthread_work+0x40/0x40 [ 868.192766] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.192767] [] ? insert_kthread_work+0x40/0x40 [ 868.192768] Task dump for CPU 20: [ 868.192769] Task dump for CPU 2: [ 868.192770] ll_mgs_0013 R [ 868.192771] mdt00_013 R [ 868.192772] running task [ 868.192772] running task [ 868.192774] 0 10822 2 0x00000088 [ 868.192775] 0 17594 2 0x00000088 [ 868.192776] Call Trace: [ 868.192777] Call Trace: [ 868.192779] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.192780] [ 868.192782] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.192785] [] sched_show_task+0xbf/0x120 [ 868.192787] [] ? _raw_spin_lock+0x30/0x40 [ 868.192789] [] dump_cpu_task+0x39/0x70 [ 868.192795] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.192797] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.192806] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.192808] [] rcu_check_callbacks+0x482/0x770 [ 868.192810] [] ? wake_up_atomic_t+0x30/0x30 [ 868.192812] [] update_process_times+0x46/0x80 [ 868.192820] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.192823] [] tick_sched_handle+0x30/0x70 [ 868.192847] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.192849] [] tick_sched_timer+0x39/0x80 [ 868.192870] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.192872] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.192874] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.192876] [] ? tick_sched_do_timer+0x50/0x50 [ 868.192878] [] ? check_preempt_curr+0x80/0xa0 [ 868.192879] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.192881] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.192883] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.192913] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.192915] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.192943] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.192945] [] apic_timer_interrupt+0x16a/0x170 [ 868.192947] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.192948] [ 868.192973] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.192975] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.192999] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.193002] [] queued_spin_lock_slowpath+0xb/0xf [ 868.193004] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.193006] [] _raw_spin_lock+0x30/0x40 [ 868.193008] [] ? sched_feat_set+0xf0/0xf0 [ 868.193014] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.193038] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.193043] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.193045] [] ? __switch_to+0xce/0x5a0 [ 868.193051] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.193075] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.193077] [] ? wake_up_atomic_t+0x30/0x30 [ 868.193079] [] ? kthread+0xd1/0xe0 [ 868.193085] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.193088] [] ? insert_kthread_work+0x40/0x40 [ 868.193106] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.193109] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.193125] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.193127] [] ? insert_kthread_work+0x40/0x40 [ 868.193145] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.193146] Task dump for CPU 23: [ 868.193148] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.193149] ll_mgs_0014 R [ 868.193151] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.193152] running task [ 868.193153] [] ? check_preempt_curr+0x80/0xa0 [ 868.193155] 0 10826 2 0x00000088 [ 868.193157] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.193158] Call Trace: [ 868.193181] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.193183] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.193205] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.193207] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.193209] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.193211] [] ? _raw_spin_lock+0x30/0x40 [ 868.193231] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.193237] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.193256] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.193264] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.193266] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.193268] [] ? wake_up_atomic_t+0x30/0x30 [ 868.193270] [] ? sched_feat_set+0xf0/0xf0 [ 868.193278] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.193296] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.193320] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.193322] [] ? __switch_to+0xce/0x5a0 [ 868.193346] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.193364] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.193367] [] ? check_preempt_curr+0x80/0xa0 [ 868.193369] [] kthread+0xd1/0xe0 [ 868.193392] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.193394] [] ? insert_kthread_work+0x40/0x40 [ 868.193417] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.193419] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.193449] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.193451] [] ? insert_kthread_work+0x40/0x40 [ 868.193479] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.193481] Task dump for CPU 8: [ 868.193482] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.193483] mdt01_022 R [ 868.193509] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.193510] running task [ 868.193535] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.193536] 0 17601 2 0x00000088 [ 868.193538] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.193539] Call Trace: [ 868.193540] [] ? sched_feat_set+0xf0/0xf0 [ 868.193542] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.193567] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.193569] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.193571] [] ? __switch_to+0xce/0x5a0 [ 868.193573] [] ? _raw_spin_lock+0x30/0x40 [ 868.193597] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.193603] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.193605] [] ? kthread+0xd1/0xe0 [ 868.193612] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.193614] [] ? insert_kthread_work+0x40/0x40 [ 868.193616] [] ? wake_up_atomic_t+0x30/0x30 [ 868.193618] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.193624] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.193626] [] ? insert_kthread_work+0x40/0x40 [ 868.193644] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.193645] Task dump for CPU 29: [ 868.193663] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.193665] mdt01_017 R [ 868.193683] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.193684] running task [ 868.193701] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.193702] 0 17555 2 0x00000088 [ 868.193719] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.193720] Call Trace: [ 868.193743] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.193746] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.193767] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.193769] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.193771] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.193773] [] ? _raw_spin_lock+0x30/0x40 [ 868.193792] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.193799] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.193817] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.193826] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.193827] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.193830] [] ? wake_up_atomic_t+0x30/0x30 [ 868.193831] [] ? sched_feat_set+0xf0/0xf0 [ 868.193839] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.193857] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.193881] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.193899] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.193925] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.193927] [] ? kthread+0xd1/0xe0 [ 868.193929] [] ? check_preempt_curr+0x80/0xa0 [ 868.193931] [] ? insert_kthread_work+0x40/0x40 [ 868.193954] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.193956] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.193979] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.193981] [] ? insert_kthread_work+0x40/0x40 [ 868.194011] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.194012] Task dump for CPU 13: [ 868.194040] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.194041] mdt01_003 R [ 868.194043] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.194044] running task [ 868.194081] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.194082] 0 10825 2 0x00000088 [ 868.194120] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.194121] Call Trace: [ 868.194124] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.194126] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.194128] [] ? sched_feat_set+0xf0/0xf0 [ 868.194130] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.194168] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.194170] [] ? _raw_spin_lock+0x30/0x40 [ 868.194208] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.194213] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.194215] [] ? kthread+0xd1/0xe0 [ 868.194222] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.194224] [] ? insert_kthread_work+0x40/0x40 [ 868.194226] [] ? wake_up_atomic_t+0x30/0x30 [ 868.194228] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.194234] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.194237] [] ? insert_kthread_work+0x40/0x40 [ 868.194254] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.194255] Task dump for CPU 30: [ 868.194271] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.194273] mdt01_018 R [ 868.194274] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.194275] running task [ 868.194277] [] ? check_preempt_curr+0x80/0xa0 [ 868.194279] 0 17592 2 0x00000088 [ 868.194281] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.194282] Call Trace: [ 868.194305] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.194307] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.194328] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.194331] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.194333] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.194335] [] ? _raw_spin_lock+0x30/0x40 [ 868.194355] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.194363] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.194382] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.194392] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.194393] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.194396] [] ? wake_up_atomic_t+0x30/0x30 [ 868.194397] [] ? sched_feat_set+0xf0/0xf0 [ 868.194407] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.194426] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.194462] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.194480] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.194514] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.194517] [] ? kthread+0xd1/0xe0 [ 868.194519] [] ? check_preempt_curr+0x80/0xa0 [ 868.194521] [] ? insert_kthread_work+0x40/0x40 [ 868.194523] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.194525] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.194565] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.194567] [] ? insert_kthread_work+0x40/0x40 [ 868.194606] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.194607] Task dump for CPU 14: [ 868.194609] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.194610] mdt01_016 R [ 868.194648] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.194649] running task [ 868.194687] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.194688] 0 17554 2 0x00000088 [ 868.194690] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.194691] Call Trace: [ 868.194693] [] ? sched_feat_set+0xf0/0xf0 [ 868.194695] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.194733] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.194735] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.194772] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.194774] [] ? _raw_spin_lock+0x30/0x40 [ 868.194776] [] ? kthread+0xd1/0xe0 [ 868.194781] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.194783] [] ? insert_kthread_work+0x40/0x40 [ 868.194790] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.194792] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.194794] [] ? wake_up_atomic_t+0x30/0x30 [ 868.194796] [] ? insert_kthread_work+0x40/0x40 [ 868.194802] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.194820] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.194838] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.194839] [] ? check_preempt_curr+0x80/0xa0 [ 868.194856] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.194873] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.194895] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.194916] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.194917] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.194937] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.194955] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.194956] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.194957] [] ? sched_feat_set+0xf0/0xf0 [ 868.194975] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.194993] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.194995] [] ? kthread+0xd1/0xe0 [ 868.194996] [] ? insert_kthread_work+0x40/0x40 [ 868.194997] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.194998] [] ? insert_kthread_work+0x40/0x40 [ 868.194999] Task dump for CPU 15: [ 868.194999] mdt01_007 R [ 868.195000] running task [ 868.195000] 0 17310 2 0x00000088 [ 868.195001] Call Trace: [ 868.195002] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.195004] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.195005] [] ? _raw_spin_lock+0x30/0x40 [ 868.195010] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.195017] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.195018] [] ? wake_up_atomic_t+0x30/0x30 [ 868.195024] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.195041] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.195058] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.195061] [] ? check_preempt_curr+0x80/0xa0 [ 868.195078] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.195094] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.195117] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.195137] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.195138] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.195157] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.195175] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.195176] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.195177] [] ? sched_feat_set+0xf0/0xf0 [ 868.195195] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.195212] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.195214] [] ? kthread+0xd1/0xe0 [ 868.195215] [] ? insert_kthread_work+0x40/0x40 [ 868.195216] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.195218] [] ? insert_kthread_work+0x40/0x40 [ 868.195219] Task dump for CPU 20: [ 868.195219] Task dump for CPU 2: [ 868.195220] ll_mgs_0013 R [ 868.195221] mdt00_013 R [ 868.195222] running task [ 868.195222] running task [ 868.195224] 0 10822 2 0x00000088 [ 868.195225] 0 17594 2 0x00000088 [ 868.195226] Call Trace: [ 868.195227] Call Trace: [ 868.195229] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.195231] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.195234] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.195236] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.195238] [] ? _raw_spin_lock+0x30/0x40 [ 868.195239] [] ? _raw_spin_lock+0x30/0x40 [ 868.195244] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.195249] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.195256] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.195260] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.195262] [] ? wake_up_atomic_t+0x30/0x30 [ 868.195269] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.195274] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.195277] [] ? wake_up_atomic_t+0x30/0x30 [ 868.195294] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.195300] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.195316] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.195335] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.195337] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.195353] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.195355] [] ? check_preempt_curr+0x80/0xa0 [ 868.195373] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.195375] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.195376] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.195399] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.195401] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.195422] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.195424] [] ? check_preempt_curr+0x80/0xa0 [ 868.195426] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.195427] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.195446] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.195469] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.195488] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.195509] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.195511] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.195513] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.195515] [] ? sched_feat_set+0xf0/0xf0 [ 868.195534] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.195552] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.195571] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.195573] [] ? __switch_to+0xce/0x5a0 [ 868.195575] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.195593] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.195594] [] ? sched_feat_set+0xf0/0xf0 [ 868.195597] [] ? kthread+0xd1/0xe0 [ 868.195615] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.195617] [] ? insert_kthread_work+0x40/0x40 [ 868.195619] [] ? __switch_to+0xce/0x5a0 [ 868.195621] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.195639] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.195640] [] ? insert_kthread_work+0x40/0x40 [ 868.195642] [] ? kthread+0xd1/0xe0 [ 868.195643] Task dump for CPU 23: [ 868.195645] [] ? insert_kthread_work+0x40/0x40 [ 868.195646] ll_mgs_0014 R [ 868.195648] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.195649] running task [ 868.195650] [] ? insert_kthread_work+0x40/0x40 [ 868.195651] 0 10826 2 0x00000088 [ 868.195652] Task dump for CPU 8: [ 868.195652] Call Trace: [ 868.195654] mdt01_022 R [ 868.195656] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.195657] running task [ 868.195658] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.195660] 0 17601 2 0x00000088 [ 868.195661] [] ? _raw_spin_lock+0x30/0x40 [ 868.195662] Call Trace: [ 868.195667] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.195669] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.195676] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.195678] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.195680] [] ? wake_up_atomic_t+0x30/0x30 [ 868.195682] [] ? _raw_spin_lock+0x30/0x40 [ 868.195688] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.195693] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.195710] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.195717] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.195734] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.195736] [] ? wake_up_atomic_t+0x30/0x30 [ 868.195738] [] ? check_preempt_curr+0x80/0xa0 [ 868.195744] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.195761] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.195779] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.195796] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.195814] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.195837] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.195856] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.195877] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.195901] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.195902] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.195926] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.195946] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.195974] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.195992] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.196019] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.196021] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.196022] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.196024] [] ? sched_feat_set+0xf0/0xf0 [ 868.196044] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.196062] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.196081] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.196083] [] ? __switch_to+0xce/0x5a0 [ 868.196085] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.196102] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.196104] [] ? sched_feat_set+0xf0/0xf0 [ 868.196106] [] ? kthread+0xd1/0xe0 [ 868.196124] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.196126] [] ? insert_kthread_work+0x40/0x40 [ 868.196144] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.196146] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.196148] [] ? kthread+0xd1/0xe0 [ 868.196150] [] ? insert_kthread_work+0x40/0x40 [ 868.196151] [] ? insert_kthread_work+0x40/0x40 [ 868.196152] Task dump for CPU 29: [ 868.196154] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.196155] mdt01_017 R [ 868.196157] [] ? insert_kthread_work+0x40/0x40 [ 868.196158] running task [ 868.196158] Task dump for CPU 13: [ 868.196160] 0 17555 2 0x00000088 [ 868.196161] mdt01_003 R [ 868.196162] Call Trace: [ 868.196163] running task [ 868.196165] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.196166] 0 10825 2 0x00000088 [ 868.196168] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.196169] Call Trace: [ 868.196171] [] ? _raw_spin_lock+0x30/0x40 [ 868.196173] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.196178] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.196180] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.196186] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.196188] [] ? _raw_spin_lock+0x30/0x40 [ 868.196190] [] ? wake_up_atomic_t+0x30/0x30 [ 868.196196] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.196201] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.196208] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.196225] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.196227] [] ? wake_up_atomic_t+0x30/0x30 [ 868.196245] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.196251] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.196253] [] ? check_preempt_curr+0x80/0xa0 [ 868.196271] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.196288] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.196304] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.196321] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.196323] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.196345] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.196347] [] ? check_preempt_curr+0x80/0xa0 [ 868.196368] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.196369] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.196371] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.196396] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.196415] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.196442] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.196461] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.196463] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.196464] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.196490] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.196491] [] ? sched_feat_set+0xf0/0xf0 [ 868.196517] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.196535] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.196537] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.196555] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.196556] [] ? sched_feat_set+0xf0/0xf0 [ 868.196558] [] ? kthread+0xd1/0xe0 [ 868.196584] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.196586] [] ? insert_kthread_work+0x40/0x40 [ 868.196611] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.196613] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.196615] [] ? kthread+0xd1/0xe0 [ 868.196617] [] ? insert_kthread_work+0x40/0x40 [ 868.196619] [] ? insert_kthread_work+0x40/0x40 [ 868.196620] Task dump for CPU 30: [ 868.196622] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.196623] mdt01_018 R [ 868.196625] [] ? insert_kthread_work+0x40/0x40 [ 868.196625] running task [ 868.196626] Task dump for CPU 14: [ 868.196628] 0 17592 2 0x00000088 [ 868.196628] mdt01_016 R [ 868.196629] Call Trace: [ 868.196630] running task [ 868.196632] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.196633] 0 17554 2 0x00000088 [ 868.196635] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.196636] Call Trace: [ 868.196638] [] ? _raw_spin_lock+0x30/0x40 [ 868.196640] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.196646] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.196648] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.196654] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.196656] [] ? _raw_spin_lock+0x30/0x40 [ 868.196658] [] ? wake_up_atomic_t+0x30/0x30 [ 868.196664] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.196670] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.196677] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.196695] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.196697] [] ? wake_up_atomic_t+0x30/0x30 [ 868.196713] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.196720] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.196722] [] ? check_preempt_curr+0x80/0xa0 [ 868.196746] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.196748] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.196772] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.196795] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.196797] [] ? check_preempt_curr+0x80/0xa0 [ 868.196818] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.196842] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.196843] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.196867] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.196887] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.196914] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.196932] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.196959] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.196961] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.196963] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.196964] [] ? sched_feat_set+0xf0/0xf0 [ 868.196990] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.197009] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.197030] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.197047] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.197049] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.197052] [] ? kthread+0xd1/0xe0 [ 868.197053] [] ? sched_feat_set+0xf0/0xf0 [ 868.197055] [] ? insert_kthread_work+0x40/0x40 [ 868.197074] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.197076] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.197094] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.197096] [] ? insert_kthread_work+0x40/0x40 [ 868.197098] [] ? kthread+0xd1/0xe0 [ 868.197099] [] ? insert_kthread_work+0x40/0x40 [ 868.197101] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.197102] [] ? insert_kthread_work+0x40/0x40 [ 868.197102] Task dump for CPU 15: [ 868.197103] mdt01_007 R [ 868.197103] running task [ 868.197104] 0 17310 2 0x00000088 [ 868.197104] Call Trace: [ 868.197105] [ 868.197106] [] sched_show_task+0xbf/0x120 [ 868.197108] [] dump_cpu_task+0x39/0x70 [ 868.197109] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.197111] [] rcu_check_callbacks+0x482/0x770 [ 868.197113] [] update_process_times+0x46/0x80 [ 868.197115] [] tick_sched_handle+0x30/0x70 [ 868.197116] [] tick_sched_timer+0x39/0x80 [ 868.197117] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.197119] [] ? tick_sched_do_timer+0x50/0x50 [ 868.197120] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.197122] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.197123] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.197125] [] apic_timer_interrupt+0x16a/0x170 [ 868.197125] [ 868.197127] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.197128] [] queued_spin_lock_slowpath+0xb/0xf [ 868.197130] [] _raw_spin_lock+0x30/0x40 [ 868.197134] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.197141] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.197142] [] ? wake_up_atomic_t+0x30/0x30 [ 868.197148] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.197165] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.197183] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.197185] [] ? check_preempt_curr+0x80/0xa0 [ 868.197201] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.197218] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.197241] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.197262] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.197263] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.197282] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.197300] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.197301] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.197302] [] ? sched_feat_set+0xf0/0xf0 [ 868.197320] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.197337] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.197339] [] kthread+0xd1/0xe0 [ 868.197340] [] ? insert_kthread_work+0x40/0x40 [ 868.197342] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.197343] [] ? insert_kthread_work+0x40/0x40 [ 868.197344] Task dump for CPU 20: [ 868.197344] Task dump for CPU 2: [ 868.197345] ll_mgs_0013 R [ 868.197346] mdt00_013 R [ 868.197346] running task [ 868.197346] running task [ 868.197347] 0 10822 2 0x00000088 [ 868.197348] 0 17594 2 0x00000088 [ 868.197348] Call Trace: [ 868.197349] Call Trace: [ 868.197350] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.197352] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.197354] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.197356] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.197357] [] ? _raw_spin_lock+0x30/0x40 [ 868.197358] [] ? _raw_spin_lock+0x30/0x40 [ 868.197363] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.197368] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.197374] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.197378] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.197380] [] ? wake_up_atomic_t+0x30/0x30 [ 868.197385] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.197391] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.197392] [] ? wake_up_atomic_t+0x30/0x30 [ 868.197409] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.197415] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.197430] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.197448] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.197450] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.197465] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.197467] [] ? check_preempt_curr+0x80/0xa0 [ 868.197484] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.197485] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.197487] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.197509] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.197510] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.197531] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.197532] [] ? check_preempt_curr+0x80/0xa0 [ 868.197533] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.197534] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.197553] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.197576] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.197593] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.197614] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.197616] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.197617] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.197618] [] ? sched_feat_set+0xf0/0xf0 [ 868.197637] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.197655] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.197673] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.197674] [] ? __switch_to+0xce/0x5a0 [ 868.197676] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.197693] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.197694] [] ? sched_feat_set+0xf0/0xf0 [ 868.197696] [] ? kthread+0xd1/0xe0 [ 868.197713] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.197715] [] ? insert_kthread_work+0x40/0x40 [ 868.197716] [] ? __switch_to+0xce/0x5a0 [ 868.197718] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.197735] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.197737] [] ? insert_kthread_work+0x40/0x40 [ 868.197738] [] ? kthread+0xd1/0xe0 [ 868.197739] Task dump for CPU 23: [ 868.197740] [] ? insert_kthread_work+0x40/0x40 [ 868.197741] ll_mgs_0014 R [ 868.197742] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.197742] running task [ 868.197744] [] ? insert_kthread_work+0x40/0x40 [ 868.197744] 0 10826 2 0x00000088 [ 868.197745] Task dump for CPU 8: [ 868.197745] Call Trace: [ 868.197746] mdt01_022 R [ 868.197747] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.197747] running task [ 868.197749] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.197750] 0 17601 2 0x00000088 [ 868.197751] [] ? _raw_spin_lock+0x30/0x40 [ 868.197751] Call Trace: [ 868.197756] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.197756] [ 868.197762] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.197764] [] sched_show_task+0xbf/0x120 [ 868.197765] [] ? wake_up_atomic_t+0x30/0x30 [ 868.197767] [] dump_cpu_task+0x39/0x70 [ 868.197772] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.197774] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.197791] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.197792] [] rcu_check_callbacks+0x482/0x770 [ 868.197809] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.197811] [] update_process_times+0x46/0x80 [ 868.197812] [] ? check_preempt_curr+0x80/0xa0 [ 868.197814] [] tick_sched_handle+0x30/0x70 [ 868.197831] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.197832] [] tick_sched_timer+0x39/0x80 [ 868.197848] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.197849] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.197871] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.197873] [] ? tick_sched_do_timer+0x50/0x50 [ 868.197893] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.197895] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.197896] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.197897] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.197915] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.197917] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.197935] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.197936] [] apic_timer_interrupt+0x16a/0x170 [ 868.197937] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.197937] [ 868.197938] [] ? sched_feat_set+0xf0/0xf0 [ 868.197940] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.197957] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.197959] [] queued_spin_lock_slowpath+0xb/0xf [ 868.197960] [] ? __switch_to+0xce/0x5a0 [ 868.197962] [] _raw_spin_lock+0x30/0x40 [ 868.197987] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.197991] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.197993] [] ? kthread+0xd1/0xe0 [ 868.197999] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.198000] [] ? insert_kthread_work+0x40/0x40 [ 868.198002] [] ? wake_up_atomic_t+0x30/0x30 [ 868.198003] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.198009] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.198010] [] ? insert_kthread_work+0x40/0x40 [ 868.198029] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.198029] Task dump for CPU 29: [ 868.198030] mdt01_017 R [ 868.198053] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.198054] running task [ 868.198054] 0 17555 2 0x00000088 [ 868.198078] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.198079] Call Trace: [ 868.198080] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.198104] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.198105] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.198129] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.198130] [] ? _raw_spin_lock+0x30/0x40 [ 868.198157] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.198162] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.198188] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.198194] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.198195] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.198197] [] ? wake_up_atomic_t+0x30/0x30 [ 868.198222] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.198227] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.198252] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.198270] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.198271] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.198288] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.198289] [] ? sched_feat_set+0xf0/0xf0 [ 868.198290] [] ? check_preempt_curr+0x80/0xa0 [ 868.198315] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.198332] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.198357] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.198374] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.198375] [] kthread+0xd1/0xe0 [ 868.198398] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.198399] [] ? insert_kthread_work+0x40/0x40 [ 868.198419] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.198421] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.198422] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.198424] [] ? insert_kthread_work+0x40/0x40 [ 868.198442] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.198443] Task dump for CPU 13: [ 868.198460] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.198461] mdt01_003 R [ 868.198462] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.198463] running task [ 868.198464] [] ? sched_feat_set+0xf0/0xf0 [ 868.198464] 0 10825 2 0x00000088 [ 868.198482] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.198483] Call Trace: [ 868.198484] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.198502] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.198503] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.198505] [] ? kthread+0xd1/0xe0 [ 868.198506] [] ? _raw_spin_lock+0x30/0x40 [ 868.198508] [] ? insert_kthread_work+0x40/0x40 [ 868.198513] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.198514] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.198521] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.198522] [] ? insert_kthread_work+0x40/0x40 [ 868.198523] [] ? wake_up_atomic_t+0x30/0x30 [ 868.198524] Task dump for CPU 30: [ 868.198530] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.198531] mdt01_018 R [ 868.198554] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.198555] running task [ 868.198576] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.198577] 0 17592 2 0x00000088 [ 868.198578] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.198579] Call Trace: [ 868.198580] [] ? check_preempt_curr+0x80/0xa0 [ 868.198581] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.198583] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.198584] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.198611] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.198612] [] ? _raw_spin_lock+0x30/0x40 [ 868.198639] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.198643] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.198644] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.198650] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.198675] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.198677] [] ? wake_up_atomic_t+0x30/0x30 [ 868.198702] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.198707] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.198708] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.198726] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.198727] [] ? sched_feat_set+0xf0/0xf0 [ 868.198742] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.198767] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.198769] [] ? check_preempt_curr+0x80/0xa0 [ 868.198794] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.198795] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.198796] [] ? kthread+0xd1/0xe0 [ 868.198819] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.198820] [] ? insert_kthread_work+0x40/0x40 [ 868.198841] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.198842] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.198843] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.198845] [] ? insert_kthread_work+0x40/0x40 [ 868.198863] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.198864] Task dump for CPU 14: [ 868.198881] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.198882] mdt01_016 R [ 868.198883] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.198883] running task [ 868.198884] [] ? sched_feat_set+0xf0/0xf0 [ 868.198885] 0 17554 2 0x00000088 [ 868.198903] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.198903] Call Trace: [ 868.198905] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.198922] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.198924] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.198925] [] ? kthread+0xd1/0xe0 [ 868.198927] [] ? _raw_spin_lock+0x30/0x40 [ 868.198928] [] ? insert_kthread_work+0x40/0x40 [ 868.198933] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.198934] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.198941] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.198942] [] ? insert_kthread_work+0x40/0x40 [ 868.198944] [] ? wake_up_atomic_t+0x30/0x30 [ 868.198950] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.198974] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.198998] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.198999] [] ? check_preempt_curr+0x80/0xa0 [ 868.199020] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.199039] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.199061] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.199082] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.199084] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.199104] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.199122] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.199123] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.199124] [] ? sched_feat_set+0xf0/0xf0 [ 868.199142] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.199160] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.199162] [] ? kthread+0xd1/0xe0 [ 868.199163] [] ? insert_kthread_work+0x40/0x40 [ 868.199164] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.199166] [] ? insert_kthread_work+0x40/0x40 [ 868.199166] Task dump for CPU 15: [ 868.199167] mdt01_007 R [ 868.199167] running task [ 868.199167] 0 17310 2 0x00000088 [ 868.199168] Call Trace: [ 868.199169] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.199171] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.199172] [] ? _raw_spin_lock+0x30/0x40 [ 868.199177] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.199183] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.199184] [] ? wake_up_atomic_t+0x30/0x30 [ 868.199190] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.199207] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.199226] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.199227] [] ? check_preempt_curr+0x80/0xa0 [ 868.199251] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.199274] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.199302] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.199328] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.199329] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.199355] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.199380] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.199381] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.199382] [] ? sched_feat_set+0xf0/0xf0 [ 868.199408] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.199433] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.199435] [] ? kthread+0xd1/0xe0 [ 868.199436] [] ? insert_kthread_work+0x40/0x40 [ 868.199437] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.199439] [] ? insert_kthread_work+0x40/0x40 [ 868.199439] Task dump for CPU 20: [ 868.199440] Task dump for CPU 2: [ 868.199441] ll_mgs_0013 R [ 868.199442] mdt00_013 R [ 868.199443] running task [ 868.199444] running task [ 868.199445] 0 10822 2 0x00000088 [ 868.199446] 0 17594 2 0x00000088 [ 868.199447] Call Trace: [ 868.199448] Call Trace: [ 868.199450] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.199453] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.199455] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.199458] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.199460] [] ? _raw_spin_lock+0x30/0x40 [ 868.199462] [] ? _raw_spin_lock+0x30/0x40 [ 868.199467] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.199474] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 868.199481] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.199487] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 868.199489] [] ? wake_up_atomic_t+0x30/0x30 [ 868.199498] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 868.199505] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.199508] [] ? wake_up_atomic_t+0x30/0x30 [ 868.199532] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.199540] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.199562] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.199588] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.199590] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.199613] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.199614] [] ? check_preempt_curr+0x80/0xa0 [ 868.199639] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 868.199641] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.199644] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.199671] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.199674] [] ? sched_clock_cpu+0xa5/0xe0 [ 868.199700] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.199703] [] ? check_preempt_curr+0x80/0xa0 [ 868.199704] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.199707] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.199732] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.199763] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.199789] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.199818] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.199820] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.199822] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.199824] [] ? sched_feat_set+0xf0/0xf0 [ 868.199850] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.199876] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.199902] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.199903] [] ? __switch_to+0xce/0x5a0 [ 868.199906] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.199931] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.199933] [] ? sched_feat_set+0xf0/0xf0 [ 868.199935] [] ? kthread+0xd1/0xe0 [ 868.199960] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.199962] [] ? insert_kthread_work+0x40/0x40 [ 868.199964] [] ? __switch_to+0xce/0x5a0 [ 868.199966] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.199990] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.199992] [] ? insert_kthread_work+0x40/0x40 [ 868.199995] [] ? kthread+0xd1/0xe0 [ 868.199995] Task dump for CPU 23: [ 868.199998] [] ? insert_kthread_work+0x40/0x40 [ 868.199999] ll_mgs_0014 R [ 868.200001] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.200002] running task [ 868.200004] [] ? insert_kthread_work+0x40/0x40 [ 868.200005] 0 10826 2 0x00000088 [ 868.200006] Task dump for CPU 8: [ 868.200007] Call Trace: [ 868.200008] mdt01_022 R [ 868.200011] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.200012] running task [ 868.200014] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.200015] 0 17601 2 0x00000088 [ 868.200017] [] ? _raw_spin_lock+0x30/0x40 [ 868.200018] Call Trace: [ 868.200022] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.200025] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.200031] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.200034] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.200035] [] ? wake_up_atomic_t+0x30/0x30 [ 868.200038] [] ? _raw_spin_lock+0x30/0x40 [ 868.200043] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.200050] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.200068] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.200077] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.200095] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.200097] [] ? wake_up_atomic_t+0x30/0x30 [ 868.200099] [] ? check_preempt_curr+0x80/0xa0 [ 868.200107] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.200125] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.200149] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.200165] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.200190] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.200213] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.200239] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 868.200261] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.200286] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.200288] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.200312] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.200331] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.200362] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.200381] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.200410] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.200412] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.200414] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.200415] [] ? sched_feat_set+0xf0/0xf0 [ 868.200442] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.200466] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.200492] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.200494] [] ? __switch_to+0xce/0x5a0 [ 868.200496] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.200522] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.200524] [] ? sched_feat_set+0xf0/0xf0 [ 868.200526] [] ? kthread+0xd1/0xe0 [ 868.200552] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.200553] [] ? insert_kthread_work+0x40/0x40 [ 868.200579] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.200580] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.200583] [] ? kthread+0xd1/0xe0 [ 868.200585] [] ? insert_kthread_work+0x40/0x40 [ 868.200587] [] ? insert_kthread_work+0x40/0x40 [ 868.200588] Task dump for CPU 29: [ 868.200590] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.200591] mdt01_017 R [ 868.200593] [] ? insert_kthread_work+0x40/0x40 [ 868.200593] running task [ 868.200594] 0 17555 2 0x00000088 [ 868.200595] Task dump for CPU 13: [ 868.200595] Call Trace: [ 868.200597] mdt01_003 R [ 868.200599] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.200600] running task [ 868.200602] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.200603] 0 10825 2 0x00000088 [ 868.200605] [] ? _raw_spin_lock+0x30/0x40 [ 868.200606] Call Trace: [ 868.200611] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.200614] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.200621] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.200623] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.200625] [] ? wake_up_atomic_t+0x30/0x30 [ 868.200627] [] ? _raw_spin_lock+0x30/0x40 [ 868.200634] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.200640] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.200664] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.200673] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.200697] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.200700] [] ? wake_up_atomic_t+0x30/0x30 [ 868.200702] [] ? check_preempt_curr+0x80/0xa0 [ 868.200710] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.200734] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.200758] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.200782] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.200805] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.200832] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.200834] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.200861] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.200864] [] ? check_preempt_curr+0x80/0xa0 [ 868.200865] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.200867] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.200893] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.200924] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.200949] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.200978] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.200980] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.200982] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.200984] [] ? sched_feat_set+0xf0/0xf0 [ 868.201011] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.201031] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.201057] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.201076] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.201079] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.201081] [] ? kthread+0xd1/0xe0 [ 868.201083] [] ? sched_feat_set+0xf0/0xf0 [ 868.201085] [] ? insert_kthread_work+0x40/0x40 [ 868.201110] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.201113] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.201138] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.201140] [] ? insert_kthread_work+0x40/0x40 [ 868.201143] [] ? kthread+0xd1/0xe0 [ 868.201143] Task dump for CPU 30: [ 868.201146] [] ? insert_kthread_work+0x40/0x40 [ 868.201147] mdt01_018 R [ 868.201149] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.201150] running task [ 868.201152] [] ? insert_kthread_work+0x40/0x40 [ 868.201154] 0 17592 2 0x00000088 [ 868.201155] Task dump for CPU 14: [ 868.201156] Call Trace: [ 868.201157] mdt01_016 R [ 868.201159] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.201160] running task [ 868.201162] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.201163] 0 17554 2 0x00000088 [ 868.201164] [] ? _raw_spin_lock+0x30/0x40 [ 868.201165] Call Trace: [ 868.201171] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.201173] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.201180] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.201182] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.201184] [] ? wake_up_atomic_t+0x30/0x30 [ 868.201186] [] ? _raw_spin_lock+0x30/0x40 [ 868.201192] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.201198] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.201216] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.201225] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.201241] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.201243] [] ? wake_up_atomic_t+0x30/0x30 [ 868.201245] [] ? check_preempt_curr+0x80/0xa0 [ 868.201254] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.201255] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.201279] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.201302] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.201327] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.201349] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.201351] [] ? check_preempt_curr+0x80/0xa0 [ 868.201353] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.201377] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.201396] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.201421] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.201439] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.201470] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.201471] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.201500] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.201501] [] ? sched_feat_set+0xf0/0xf0 [ 868.201504] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.201522] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.201548] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.201566] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.201592] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.201594] [] ? kthread+0xd1/0xe0 [ 868.201596] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.201598] [] ? insert_kthread_work+0x40/0x40 [ 868.201600] [] ? sched_feat_set+0xf0/0xf0 [ 868.201601] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.201626] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.201628] [] ? insert_kthread_work+0x40/0x40 [ 868.201652] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.201654] [] ? kthread+0xd1/0xe0 [ 868.201656] [] ? insert_kthread_work+0x40/0x40 [ 868.201658] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.201659] [] ? insert_kthread_work+0x40/0x40 [ 868.201660] Task dump for CPU 15: [ 868.201661] mdt01_007 R [ 868.201661] running task [ 868.201662] 0 17310 2 0x00000088 [ 868.201662] Call Trace: [ 868.201664] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 868.201666] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.201668] [] ? _raw_spin_lock+0x30/0x40 [ 868.201674] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.201683] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.201685] [] ? wake_up_atomic_t+0x30/0x30 [ 868.201693] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.201716] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.201740] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.201742] [] ? check_preempt_curr+0x80/0xa0 [ 868.201766] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.201790] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.201820] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.201849] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.201851] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.201877] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.201903] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.201904] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.201906] [] ? sched_feat_set+0xf0/0xf0 [ 868.201931] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.201956] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.201958] [] ? kthread+0xd1/0xe0 [ 868.201960] [] ? insert_kthread_work+0x40/0x40 [ 868.201962] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.201964] [] ? insert_kthread_work+0x40/0x40 [ 868.201965] Task dump for CPU 20: [ 868.201965] ll_mgs_0013 R [ 868.201966] running task [ 868.201967] 0 10822 2 0x00000088 [ 868.201967] Call Trace: [ 868.201968] [ 868.201970] [] sched_show_task+0xbf/0x120 [ 868.201972] [] dump_cpu_task+0x39/0x70 [ 868.201974] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 868.201976] [] rcu_check_callbacks+0x482/0x770 [ 868.201979] [] update_process_times+0x46/0x80 [ 868.201981] [] tick_sched_handle+0x30/0x70 [ 868.201983] [] tick_sched_timer+0x39/0x80 [ 868.201984] [] __hrtimer_run_queues+0x13e/0x2f0 [ 868.201986] [] ? tick_sched_do_timer+0x50/0x50 [ 868.201988] [] hrtimer_interrupt+0xb9/0x1f0 [ 868.201989] [] local_apic_timer_interrupt+0x3b/0x60 [ 868.201991] [] smp_apic_timer_interrupt+0x43/0x60 [ 868.201993] [] apic_timer_interrupt+0x16a/0x170 [ 868.201994] [ 868.201996] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.201997] [] queued_spin_lock_slowpath+0xb/0xf [ 868.201999] [] _raw_spin_lock+0x30/0x40 [ 868.202005] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.202014] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.202016] [] ? wake_up_atomic_t+0x30/0x30 [ 868.202024] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.202047] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.202068] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.202070] [] ? enqueue_task_fair+0x208/0x6c0 [ 868.202072] [] ? check_preempt_curr+0x80/0xa0 [ 868.202074] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.202104] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.202131] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.202133] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.202159] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.202183] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.202185] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.202186] [] ? sched_feat_set+0xf0/0xf0 [ 868.202210] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.202212] [] ? __switch_to+0xce/0x5a0 [ 868.202235] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.202237] [] kthread+0xd1/0xe0 [ 868.202239] [] ? insert_kthread_work+0x40/0x40 [ 868.202241] [] ret_from_fork_nospec_begin+0x21/0x21 [ 868.202242] [] ? insert_kthread_work+0x40/0x40 [ 868.202243] Task dump for CPU 23: [ 868.202244] ll_mgs_0014 R [ 868.202244] running task [ 868.202245] 0 10826 2 0x00000088 [ 868.202245] Call Trace: [ 868.202247] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.202250] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.202251] [] ? _raw_spin_lock+0x30/0x40 [ 868.202257] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.202265] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.202267] [] ? wake_up_atomic_t+0x30/0x30 [ 868.202275] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.202298] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.202321] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.202323] [] ? check_preempt_curr+0x80/0xa0 [ 868.202347] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.202369] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.202399] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.202427] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.202429] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.202455] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.202480] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.202482] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.202483] [] ? sched_feat_set+0xf0/0xf0 [ 868.202507] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.202509] [] ? __switch_to+0xce/0x5a0 [ 868.202533] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.202535] [] ? kthread+0xd1/0xe0 [ 868.202536] [] ? insert_kthread_work+0x40/0x40 [ 868.202538] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.202540] [] ? insert_kthread_work+0x40/0x40 [ 868.202541] Task dump for CPU 29: [ 868.202541] mdt01_017 R [ 868.202541] running task [ 868.202542] 0 17555 2 0x00000088 [ 868.202543] Call Trace: [ 868.202544] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 868.202546] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.202548] [] ? _raw_spin_lock+0x30/0x40 [ 868.202554] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.202563] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.202565] [] ? wake_up_atomic_t+0x30/0x30 [ 868.202572] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.202596] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.202620] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 868.202621] [] ? check_preempt_curr+0x80/0xa0 [ 868.202644] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 868.202667] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 868.202697] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 868.202724] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.202726] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.202751] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.202775] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.202777] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.202778] [] ? sched_feat_set+0xf0/0xf0 [ 868.202802] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.202826] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.202828] [] ? kthread+0xd1/0xe0 [ 868.202829] [] ? insert_kthread_work+0x40/0x40 [ 868.202831] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.202833] [] ? insert_kthread_work+0x40/0x40 [ 868.202834] Task dump for CPU 30: [ 868.202834] mdt01_018 R [ 868.202834] running task [ 868.202835] 0 17592 2 0x00000088 [ 868.202836] Call Trace: [ 868.202838] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 868.202840] [] ? queued_spin_lock_slowpath+0xb/0xf [ 868.202841] [] ? _raw_spin_lock+0x30/0x40 [ 868.202848] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 868.202856] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 868.202858] [] ? wake_up_atomic_t+0x30/0x30 [ 868.202865] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 868.202889] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 868.202910] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 868.202912] [] ? check_preempt_curr+0x80/0xa0 [ 868.202913] [] ? ttwu_do_wakeup+0x19/0x100 [ 868.202943] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 868.202971] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 868.202972] [] ? __getnstimeofday64+0x3f/0xd0 [ 868.202997] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 868.203022] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 868.203023] [] ? __wake_up_common_lock+0x91/0xc0 [ 868.203025] [] ? sched_feat_set+0xf0/0xf0 [ 868.203049] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 868.203073] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 868.203075] [] ? kthread+0xd1/0xe0 [ 868.203076] [] ? insert_kthread_work+0x40/0x40 [ 868.203078] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 868.203080] [] ? insert_kthread_work+0x40/0x40 [ 884.443743] { [ 884.446091] 13} (t=60000 jiffies g=29074 c=29073 q=171165) [ 884.450974] Task dump for CPU 2: [ 884.455058] mdt00_013 R running task 0 17594 2 0x00000088 [ 884.463432] Call Trace: [ 884.466644] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 884.475383] [] ? queued_spin_lock_slowpath+0xb/0xf [ 884.483057] [] ? _raw_spin_lock+0x30/0x40 [ 884.489861] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 884.497723] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 884.505693] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 884.514433] [] ? wake_up_atomic_t+0x30/0x30 [ 884.521434] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 884.529031] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 884.537501] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 884.546255] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 884.554414] [] ? enqueue_task_fair+0x208/0x6c0 [ 884.561699] [] ? sched_clock_cpu+0xa5/0xe0 [ 884.568596] [] ? check_preempt_curr+0x80/0xa0 [ 884.575783] [] ? ttwu_do_wakeup+0x19/0x100 [ 884.582710] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 884.591088] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 884.600020] [] ? __getnstimeofday64+0x3f/0xd0 [ 884.607233] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 884.616479] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 884.624541] [] ? __wake_up_common_lock+0x91/0xc0 [ 884.632018] [] ? sched_feat_set+0xf0/0xf0 [ 884.638840] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 884.646515] [] ? __switch_to+0xce/0x5a0 [ 884.653144] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 884.661883] [] ? kthread+0xd1/0xe0 [ 884.668006] [] ? insert_kthread_work+0x40/0x40 [ 884.675292] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 884.683255] [] ? insert_kthread_work+0x40/0x40 [ 884.690539] Task dump for CPU 8: [ 884.694624] mdt01_022 R running task 0 17601 2 0x00000088 [ 884.702997] Call Trace: [ 884.706209] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 884.714948] [] ? queued_spin_lock_slowpath+0xb/0xf [ 884.722621] [] ? _raw_spin_lock+0x30/0x40 [ 884.729426] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 884.737299] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 884.745942] [] ? wake_up_atomic_t+0x30/0x30 [ 884.752942] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 884.760541] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 884.769023] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 884.777217] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 884.785895] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 884.793991] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 884.801509] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 884.809888] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 884.818823] [] ? __getnstimeofday64+0x3f/0xd0 [ 884.826046] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 884.835308] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 884.843369] [] ? __wake_up_common_lock+0x91/0xc0 [ 884.850848] [] ? sched_feat_set+0xf0/0xf0 [ 884.857684] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 884.865393] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 884.874132] [] ? kthread+0xd1/0xe0 [ 884.880254] [] ? insert_kthread_work+0x40/0x40 [ 884.887538] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 884.895502] [] ? insert_kthread_work+0x40/0x40 [ 884.902785] Task dump for CPU 13: [ 884.906966] mdt01_003 R running task 0 10825 2 0x00000088 [ 884.915336] Call Trace: [ 884.918548] [] sched_show_task+0xbf/0x120 [ 884.926037] [] dump_cpu_task+0x39/0x70 [ 884.932545] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 884.939637] [] rcu_check_callbacks+0x482/0x770 [ 884.946921] [] update_process_times+0x46/0x80 [ 884.954110] [] tick_sched_handle+0x30/0x70 [ 884.961008] [] tick_sched_timer+0x39/0x80 [ 884.967807] [] __hrtimer_run_queues+0x13e/0x2f0 [ 884.975188] [] ? tick_sched_do_timer+0x50/0x50 [ 884.982472] [] hrtimer_interrupt+0xb9/0x1f0 [ 884.989467] [] local_apic_timer_interrupt+0x3b/0x60 [ 884.997237] [] smp_apic_timer_interrupt+0x43/0x60 [ 885.004813] [] apic_timer_interrupt+0x16a/0x170 [ 885.012194] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 885.021622] [] queued_spin_lock_slowpath+0xb/0xf [ 885.029102] [] _raw_spin_lock+0x30/0x40 [ 885.035714] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 885.043394] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 885.051843] [] ? wake_up_atomic_t+0x30/0x30 [ 885.058846] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 885.066261] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 885.074549] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 885.083095] [] ? enqueue_task_fair+0x208/0x6c0 [ 885.090379] [] ? check_preempt_curr+0x80/0xa0 [ 885.097567] [] ? ttwu_do_wakeup+0x19/0x100 [ 885.104504] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 885.112699] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 885.121634] [] ? __getnstimeofday64+0x3f/0xd0 [ 885.128858] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 885.137925] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 885.145986] [] ? __wake_up_common_lock+0x91/0xc0 [ 885.153466] [] ? sched_feat_set+0xf0/0xf0 [ 885.160301] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 885.167818] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 885.176559] [] kthread+0xd1/0xe0 [ 885.182488] [] ? insert_kthread_work+0x40/0x40 [ 885.189772] [] ret_from_fork_nospec_begin+0x21/0x21 [ 885.197541] [] ? insert_kthread_work+0x40/0x40 [ 885.204824] Task dump for CPU 14: [ 885.209005] mdt01_016 R running task 0 17554 2 0x00000088 [ 885.217379] Call Trace: [ 885.220592] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 885.229333] [] ? queued_spin_lock_slowpath+0xb/0xf [ 885.237006] [] ? _raw_spin_lock+0x30/0x40 [ 885.243810] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 885.251686] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 885.260328] [] ? wake_up_atomic_t+0x30/0x30 [ 885.267331] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 885.274942] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 885.283426] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 885.291584] [] ? check_preempt_curr+0x80/0xa0 [ 885.298806] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 885.306903] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 885.314420] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 885.322810] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 885.331745] [] ? __getnstimeofday64+0x3f/0xd0 [ 885.338969] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 885.348230] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 885.356292] [] ? __wake_up_common_lock+0x91/0xc0 [ 885.363771] [] ? sched_feat_set+0xf0/0xf0 [ 885.370606] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 885.378316] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 885.387055] [] ? kthread+0xd1/0xe0 [ 885.393177] [] ? insert_kthread_work+0x40/0x40 [ 885.400462] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 885.408426] [] ? insert_kthread_work+0x40/0x40 [ 885.415709] Task dump for CPU 15: [ 885.419890] mdt01_007 R running task 0 17310 2 0x00000088 [ 885.428262] Call Trace: [ 885.431475] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 885.440215] [] ? queued_spin_lock_slowpath+0xb/0xf [ 885.447888] [] ? _raw_spin_lock+0x30/0x40 [ 885.454693] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 885.462569] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 885.471212] [] ? wake_up_atomic_t+0x30/0x30 [ 885.478215] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 885.485826] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 885.494309] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 885.502467] [] ? check_preempt_curr+0x80/0xa0 [ 885.509690] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 885.517786] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 885.525304] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 885.533695] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 885.542629] [] ? __getnstimeofday64+0x3f/0xd0 [ 885.549852] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 885.559114] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 885.567176] [] ? __wake_up_common_lock+0x91/0xc0 [ 885.574655] [] ? sched_feat_set+0xf0/0xf0 [ 885.581490] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 885.589200] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 885.597939] [] ? kthread+0xd1/0xe0 [ 885.604061] [] ? insert_kthread_work+0x40/0x40 [ 885.611345] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 885.619310] [] ? insert_kthread_work+0x40/0x40 [ 885.626594] Task dump for CPU 20: [ 885.630775] ll_mgs_0013 R running task 0 10822 2 0x00000088 [ 885.639147] Call Trace: [ 885.642360] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 885.651100] [] ? queued_spin_lock_slowpath+0xb/0xf [ 885.658772] [] ? _raw_spin_lock+0x30/0x40 [ 885.665579] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 885.673456] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 885.682098] [] ? wake_up_atomic_t+0x30/0x30 [ 885.689101] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 885.696712] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 885.705194] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 885.713935] [] ? enqueue_task_fair+0x208/0x6c0 [ 885.721220] [] ? check_preempt_curr+0x80/0xa0 [ 885.728407] [] ? ttwu_do_wakeup+0x19/0x100 [ 885.735343] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 885.743733] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 885.752666] [] ? __getnstimeofday64+0x3f/0xd0 [ 885.759891] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 885.769150] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 885.777212] [] ? __wake_up_common_lock+0x91/0xc0 [ 885.784692] [] ? sched_feat_set+0xf0/0xf0 [ 885.791528] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 885.799202] [] ? __switch_to+0xce/0x5a0 [ 885.805845] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 885.814586] [] ? kthread+0xd1/0xe0 [ 885.820708] [] ? insert_kthread_work+0x40/0x40 [ 885.827992] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 885.835954] [] ? insert_kthread_work+0x40/0x40 [ 885.843238] Task dump for CPU 23: [ 885.847420] ll_mgs_0014 R running task 0 10826 2 0x00000088 [ 885.855792] Call Trace: [ 885.859007] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 885.867738] [] ? queued_spin_lock_slowpath+0xb/0xf [ 885.875413] [] ? _raw_spin_lock+0x30/0x40 [ 885.882220] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 885.890096] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 885.898739] [] ? wake_up_atomic_t+0x30/0x30 [ 885.905740] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 885.913352] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 885.921837] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 885.929995] [] ? check_preempt_curr+0x80/0xa0 [ 885.937217] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 885.945312] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 885.952831] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 885.961218] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 885.970153] [] ? __getnstimeofday64+0x3f/0xd0 [ 885.977377] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 885.986638] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 885.994700] [] ? __wake_up_common_lock+0x91/0xc0 [ 886.002179] [] ? sched_feat_set+0xf0/0xf0 [ 886.009014] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 886.016686] [] ? __switch_to+0xce/0x5a0 [ 886.023329] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 886.032069] [] ? kthread+0xd1/0xe0 [ 886.038191] [] ? insert_kthread_work+0x40/0x40 [ 886.045477] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 886.053442] [] ? insert_kthread_work+0x40/0x40 [ 886.060725] Task dump for CPU 29: [ 886.064906] mdt01_017 R running task 0 17555 2 0x00000088 [ 886.073275] Call Trace: [ 886.076488] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 886.085229] [] ? queued_spin_lock_slowpath+0xb/0xf [ 886.092901] [] ? _raw_spin_lock+0x30/0x40 [ 886.099708] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 886.107581] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 886.116224] [] ? wake_up_atomic_t+0x30/0x30 [ 886.123227] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 886.130838] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 886.139322] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 886.147479] [] ? check_preempt_curr+0x80/0xa0 [ 886.154702] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 886.162797] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 886.170315] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 886.178703] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 886.187638] [] ? __getnstimeofday64+0x3f/0xd0 [ 886.194862] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 886.204122] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 886.212183] [] ? __wake_up_common_lock+0x91/0xc0 [ 886.219662] [] ? sched_feat_set+0xf0/0xf0 [ 886.226498] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 886.234207] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 886.242949] [] ? kthread+0xd1/0xe0 [ 886.249070] [] ? insert_kthread_work+0x40/0x40 [ 886.256355] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 886.264319] [] ? insert_kthread_work+0x40/0x40 [ 886.271602] Task dump for CPU 30: [ 886.275782] mdt01_018 R running task 0 17592 2 0x00000088 [ 886.284154] Call Trace: [ 886.287367] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 886.296105] [] ? queued_spin_lock_slowpath+0xb/0xf [ 886.303778] [] ? _raw_spin_lock+0x30/0x40 [ 886.310583] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 886.318459] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 886.327101] [] ? wake_up_atomic_t+0x30/0x30 [ 886.334104] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 886.341714] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 886.350196] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 886.358936] [] ? check_preempt_curr+0x80/0xa0 [ 886.366124] [] ? ttwu_do_wakeup+0x19/0x100 [ 886.373059] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 886.381448] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 886.390382] [] ? __getnstimeofday64+0x3f/0xd0 [ 886.397606] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 886.406865] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 886.414927] [] ? __wake_up_common_lock+0x91/0xc0 [ 886.422406] [] ? sched_feat_set+0xf0/0xf0 [ 886.429244] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 886.436955] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 886.445685] [] ? kthread+0xd1/0xe0 [ 886.451807] [] ? insert_kthread_work+0x40/0x40 [ 886.459091] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 886.467055] [] ? insert_kthread_work+0x40/0x40 [ 886.475223] Lustre: mdt_out: This server is not able to keep up with request traffic (cpu-bound). [ 886.475256] Lustre: 17593:0:(service.c:1542:ptlrpc_at_check_timed()) earlyQ=1 reqQ=0 recA=24, svcEst=136, delay=0 [ 886.475266] Lustre: 17593:0:(service.c:1323:ptlrpc_at_send_early_reply()) @@@ Already past deadline (-10s), not sending early reply. Consider increasing at_early_margin (5)? req@ffff97f2555e5580 x1716076184842112/t0(0) o101->a741c2cd-59a2-e958-74ff-2bedd308f0ca@192.168.131.200@o2ib39:97/0 lens 584/3272 e 0 to 0 dl 1644342532 ref 2 fl Interpret:/0/0 rc 0/0 [ 886.475268] Lustre: 17593:0:(service.c:1323:ptlrpc_at_send_early_reply()) Skipped 333 previous similar messages [ 886.475270] LNet: Service thread pid 17592 was inactive for 217.67s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [ 886.475272] Pid: 17592, comm: mdt01_018 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 SMP Tue Jan 25 12:06:24 PST 2022 [ 886.475272] Call Trace: [ 886.475281] [] 0xffffffffffffffff [ 886.475283] LustreError: dumping log to /tmp/lustre-log.1644342542.17592 [ 886.593273] Lustre: Skipped 2 previous similar messages [ 888.679102] Lustre: MGS: Received new MDS connection from 192.168.128.68@o2ib38, keep former export from same NID [ 888.691046] Lustre: Skipped 494 previous similar messages [ 889.213422] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [mdt00_009:17321] [ 889.222549] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 889.302368] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 889.363443] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 889.377805] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 889.390130] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 889.398965] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 889.410531] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 889.416941] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 889.425389] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 889.433835] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 889.442281] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 889.450727] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 889.459173] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 889.468688] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 889.475585] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 889.484032] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 889.492479] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 889.500926] Call Trace: [ 889.504141] [] queued_spin_lock_slowpath+0xb/0xf [ 889.511621] [] _raw_spin_lock+0x30/0x40 [ 889.518239] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 889.525922] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 889.534371] [] ? wake_up_atomic_t+0x30/0x30 [ 889.541370] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 889.548794] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 889.557064] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 889.565628] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 889.573787] [] ? enqueue_task_fair+0x208/0x6c0 [ 889.581072] [] ? sched_clock_cpu+0xa5/0xe0 [ 889.587968] [] ? check_preempt_curr+0x80/0xa0 [ 889.595154] [] ? ttwu_do_wakeup+0x19/0x100 [ 889.602080] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 889.610258] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 889.619183] [] ? __getnstimeofday64+0x3f/0xd0 [ 889.626390] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 889.635439] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 889.643499] [] ? __wake_up_common_lock+0x91/0xc0 [ 889.650977] [] ? sched_feat_set+0xf0/0xf0 [ 889.657796] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 889.665274] [] ? __switch_to+0xce/0x5a0 [ 889.671897] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 889.680635] [] kthread+0xd1/0xe0 [ 889.686562] [] ? insert_kthread_work+0x40/0x40 [ 889.693848] [] ret_from_fork_nospec_begin+0x21/0x21 [ 889.701617] [] ? insert_kthread_work+0x40/0x40 [ 889.708901] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 893.067866] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [mdt00_013:17594] [ 893.076899] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 893.164877] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [mdt01_022:17601] [ 893.156737] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx [ 893.164878] Modules linked in: [ 893.164879] osp(OE) [ 893.164879] mdd(OE) [ 893.164880] lod(OE) [ 893.164880] mdt(OE) [ 893.164880] lfsck(OE) [ 893.164880] mgs(OE) [ 893.164881] mgc(OE) [ 893.164881] osd_zfs(OE) [ 893.164881] lquota(OE) [ 893.164882] fid(OE) [ 893.164882] fld(OE) [ 893.164882] ptlrpc(OE) [ 893.164883] obdclass(OE) [ 893.164883] ko2iblnd(OE) [ 893.164883] lnet(OE) [ 893.164884] libcfs(OE) [ 893.164884] iTCO_wdt [ 893.164884] iTCO_vendor_support [ 893.164885] sb_edac [ 893.164885] intel_powerclamp [ 893.164885] coretemp [ 893.164886] intel_rapl [ 893.164886] iosf_mbi [ 893.164886] rpcrdma [ 893.164887] ib_iser [ 893.164887] kvm [ 893.164887] irqbypass [ 893.164888] joydev [ 893.164888] pcspkr [ 893.164888] lpc_ich [ 893.164889] i2c_i801 [ 893.164889] ioatdma [ 893.164889] ses [ 893.164890] enclosure [ 893.164890] sg [ 893.164890] ipmi_si [ 893.164891] ipmi_devintf [ 893.164891] ipmi_msghandler [ 893.164891] acpi_power_meter [ 893.164892] dm_round_robin [ 893.164892] acpi_cpufreq [ 893.164892] ib_ipoib [ 893.164893] rdma_ucm [ 893.164893] ib_umad [ 893.164893] sch_fq_codel [ 893.164893] iw_cxgb4 [ 893.164894] rdma_cm [ 893.164894] iw_cm [ 893.164894] ib_cm [ 893.164895] iw_cxgb3 [ 893.164895] zfs(POE) [ 893.164896] binfmt_misc [ 893.164896] zunicode(POE) [ 893.164896] zavl(POE) [ 893.164897] icp(POE) [ 893.164897] zcommon(POE) [ 893.164898] znvpair(POE) [ 893.164898] spl(OE) [ 893.164898] msr_safe(OE) [ 893.164899] ip_tables [ 893.164899] nfsv3 [ 893.164899] nfs_acl [ 893.164900] rpcsec_gss_krb5 [ 893.164900] auth_rpcgss [ 893.164900] nfsv4 [ 893.164901] dns_resolver [ 893.164901] nfs [ 893.164901] lockd [ 893.164902] grace [ 893.164902] fscache [ 893.164902] overlay(T) [ 893.164903] ext4 [ 893.164903] mbcache [ 893.164903] jbd2 [ 893.164904] dm_service_time [ 893.164904] sd_mod [ 893.164904] crc_t10dif [ 893.164905] crct10dif_generic [ 893.164905] mlx5_ib [ 893.164906] ib_uverbs [ 893.164906] ib_core [ 893.164906] be2iscsi [ 893.164907] bnx2i [ 893.164907] cnic [ 893.164907] uio [ 893.164908] cxgb4i [ 893.164908] cxgb4 [ 893.164908] cxgb3i [ 893.164908] cxgb3 [ 893.164909] mdio [ 893.164909] libcxgbi [ 893.164910] libcxgb [ 893.164910] qla4xxx [ 893.164910] iscsi_boot_sysfs [ 893.164911] mgag200 [ 893.164911] i2c_algo_bit [ 893.164911] 8021q [ 893.164912] drm_kms_helper [ 893.164912] garp [ 893.164912] syscopyarea [ 893.164913] mrp [ 893.164913] crct10dif_pclmul [ 893.164913] stp [ 893.164914] sysfillrect [ 893.164914] crct10dif_common [ 893.164914] sysimgblt [ 893.164915] crc32_pclmul [ 893.164915] llc [ 893.164915] fb_sys_fops [ 893.164916] crc32c_intel [ 893.164916] ttm [ 893.164916] ghash_clmulni_intel [ 893.164917] mlx5_core [ 893.164917] drm [ 893.164917] mxm_wmi [ 893.164918] ahci [ 893.164918] ixgbe(OE) [ 893.164919] aesni_intel [ 893.164919] mpt3sas [ 893.164920] mlxfw [ 893.164920] dm_multipath [ 893.164920] lrw [ 893.164921] devlink [ 893.164921] libahci [ 893.164921] gf128mul [ 893.164922] dca [ 893.164922] glue_helper [ 893.164923] ablk_helper [ 893.164923] raid_class [ 893.164923] ptp [ 893.164924] drm_panel_orientation_quirks [ 893.164924] libata [ 893.164924] cryptd [ 893.164925] scsi_transport_sas [ 893.164925] pps_core [ 893.164925] wmi [ 893.164926] sunrpc [ 893.164926] dm_mirror [ 893.164926] dm_region_hash [ 893.164927] dm_log [ 893.164927] dm_mod [ 893.164928] iscsi_tcp [ 893.164928] libiscsi_tcp [ 893.164928] libiscsi [ 893.164929] scsi_transport_iscsi [ 893.164929] fuse [ 893.164929] [ 893.164931] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.164932] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.164933] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 893.164934] RIP: 0010:[] [ 893.164936] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 893.164937] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 893.164938] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 893.164939] RDX: ffff97f2bfddb8c0 RSI: 0000000000b90001 RDI: ffff97f2a83bcf40 [ 893.164939] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 893.164940] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 893.164941] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 893.164941] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 893.164942] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.164943] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 893.164944] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.164944] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.164944] Call Trace: [ 893.164947] [] queued_spin_lock_slowpath+0xb/0xf [ 893.164950] [] _raw_spin_lock+0x30/0x40 [ 893.164957] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 893.164970] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 893.164972] [] ? wake_up_atomic_t+0x30/0x30 [ 893.164978] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.165010] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.165028] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 893.165047] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 893.165064] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 893.165081] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 893.165106] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 893.165127] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.165128] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.165148] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.165166] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.165167] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.165168] [] ? sched_feat_set+0xf0/0xf0 [ 893.165187] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.165205] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.165207] [] kthread+0xd1/0xe0 [ 893.165208] [] ? insert_kthread_work+0x40/0x40 [ 893.165210] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.165211] [] ? insert_kthread_work+0x40/0x40 [ 893.165211] Code: [ 893.165212] 0d [ 893.165212] 48 [ 893.165212] 98 [ 893.165213] 83 [ 893.165213] e2 [ 893.165213] 30 [ 893.165213] 48 [ 893.165213] 81 [ 893.165214] c2 [ 893.165214] c0 [ 893.165214] b8 [ 893.165214] 01 [ 893.165215] 00 [ 893.165215] 48 [ 893.165215] 03 [ 893.165216] 14 [ 893.165216] c5 [ 893.165216] e0 [ 893.165216] 17 [ 893.165217] d5 [ 893.165217] a0 [ 893.165217] 4c [ 893.165217] 89 [ 893.165218] 02 [ 893.165218] 41 [ 893.165218] 8b [ 893.165219] 40 [ 893.165219] 08 [ 893.165219] 85 [ 893.165219] c0 [ 893.165220] 75 [ 893.165220] 0f [ 893.165220] 0f [ 893.165220] 1f [ 893.165221] 44 [ 893.165221] 00 [ 893.165221] 00 [ 893.165222] f3 [ 893.165222] 90 [ 893.165222] 41 [ 893.165222] 8b [ 893.165223] 40 [ 893.165223] 08 [ 893.165223] <85> [ 893.165223] c0 [ 893.165224] 74 [ 893.165224] f6 [ 893.165224] 4d [ 893.165224] 8b [ 893.165225] 08 [ 893.165225] 4d [ 893.165225] 85 [ 893.165225] c9 [ 893.165226] 74 [ 893.165226] 04 [ 893.165226] 41 [ 893.165226] 0f [ 893.165227] 18 [ 893.165227] 09 [ 893.165227] 8b [ 893.165227] 17 [ 893.165228] 0f [ 893.165228] b7 [ 893.165228] c2 [ 893.165228] [ 893.191879] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [mdt01_016:17554] [ 893.191880] Modules linked in: [ 893.191880] osp(OE) [ 893.191881] mdd(OE) [ 893.191881] lod(OE) [ 893.191882] mdt(OE) [ 893.191882] lfsck(OE) [ 893.191883] mgs(OE) [ 893.191883] mgc(OE) [ 893.191883] osd_zfs(OE) [ 893.191884] lquota(OE) [ 893.191884] fid(OE) [ 893.191885] fld(OE) [ 893.191885] ptlrpc(OE) [ 893.191886] obdclass(OE) [ 893.191886] ko2iblnd(OE) [ 893.191886] lnet(OE) [ 893.191887] libcfs(OE) [ 893.191887] iTCO_wdt [ 893.191888] iTCO_vendor_support [ 893.191888] sb_edac [ 893.191888] intel_powerclamp [ 893.191889] coretemp [ 893.191889] intel_rapl [ 893.191890] iosf_mbi [ 893.191890] rpcrdma [ 893.191890] ib_iser [ 893.191891] kvm [ 893.191891] irqbypass [ 893.191891] joydev [ 893.191892] pcspkr [ 893.191892] lpc_ich [ 893.191893] i2c_i801 [ 893.191893] ioatdma [ 893.191893] ses [ 893.191894] enclosure [ 893.191894] sg [ 893.191894] ipmi_si [ 893.191895] ipmi_devintf [ 893.191895] ipmi_msghandler [ 893.191896] acpi_power_meter [ 893.191896] dm_round_robin [ 893.191896] acpi_cpufreq [ 893.191897] ib_ipoib [ 893.191897] rdma_ucm [ 893.191897] ib_umad [ 893.191898] sch_fq_codel [ 893.191898] iw_cxgb4 [ 893.191898] rdma_cm [ 893.191899] iw_cm [ 893.191899] ib_cm [ 893.191900] iw_cxgb3 [ 893.191900] zfs(POE) [ 893.191900] binfmt_misc [ 893.191901] zunicode(POE) [ 893.191901] zavl(POE) [ 893.191902] icp(POE) [ 893.191902] zcommon(POE) [ 893.191903] znvpair(POE) [ 893.191903] spl(OE) [ 893.191904] msr_safe(OE) [ 893.191904] ip_tables [ 893.191904] nfsv3 [ 893.191905] nfs_acl [ 893.191905] rpcsec_gss_krb5 [ 893.191906] auth_rpcgss [ 893.191906] nfsv4 [ 893.191907] dns_resolver [ 893.191907] nfs [ 893.191907] lockd [ 893.191908] grace [ 893.191908] fscache [ 893.191909] overlay(T) [ 893.191909] ext4 [ 893.191910] mbcache [ 893.191910] jbd2 [ 893.191910] dm_service_time [ 893.191911] sd_mod [ 893.191911] crc_t10dif [ 893.191912] crct10dif_generic [ 893.191912] mlx5_ib [ 893.191913] ib_uverbs [ 893.191913] ib_core [ 893.191913] be2iscsi [ 893.191914] bnx2i [ 893.191914] cnic [ 893.191915] uio [ 893.191915] cxgb4i [ 893.191915] cxgb4 [ 893.191916] cxgb3i [ 893.191916] cxgb3 [ 893.191917] mdio [ 893.191917] libcxgbi [ 893.191917] libcxgb [ 893.191918] qla4xxx [ 893.191918] iscsi_boot_sysfs [ 893.191919] mgag200 [ 893.191919] i2c_algo_bit [ 893.191919] 8021q [ 893.191920] drm_kms_helper [ 893.191920] garp [ 893.191921] syscopyarea [ 893.191921] mrp [ 893.191921] crct10dif_pclmul [ 893.191922] stp [ 893.191922] sysfillrect [ 893.191923] crct10dif_common [ 893.191923] sysimgblt [ 893.191924] crc32_pclmul [ 893.191924] llc [ 893.191924] fb_sys_fops [ 893.191925] crc32c_intel [ 893.191925] ttm [ 893.191926] ghash_clmulni_intel [ 893.191926] mlx5_core [ 893.191926] drm [ 893.191927] mxm_wmi [ 893.191927] ahci [ 893.191928] ixgbe(OE) [ 893.191928] aesni_intel [ 893.191928] mpt3sas [ 893.191929] mlxfw [ 893.191929] dm_multipath [ 893.191930] lrw [ 893.191930] devlink [ 893.191930] libahci [ 893.191931] gf128mul [ 893.191931] dca [ 893.191931] glue_helper [ 893.191932] ablk_helper [ 893.191932] raid_class [ 893.191933] ptp [ 893.191933] drm_panel_orientation_quirks [ 893.191933] libata [ 893.191934] cryptd [ 893.191934] scsi_transport_sas [ 893.191935] pps_core [ 893.191935] wmi [ 893.191935] sunrpc [ 893.191936] dm_mirror [ 893.191936] dm_region_hash [ 893.191937] dm_log [ 893.191937] dm_mod [ 893.191937] iscsi_tcp [ 893.191938] libiscsi_tcp [ 893.191938] libiscsi [ 893.191939] scsi_transport_iscsi [ 893.191939] fuse [ 893.191939] [ 893.191941] CPU: 14 PID: 17554 Comm: mdt01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.191942] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.191943] task: ffff98321c985280 ti: ffff98321b544000 task.ti: ffff98321b544000 [ 893.191944] RIP: 0010:[] [ 893.191946] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 893.191947] RSP: 0018:ffff98321b547ab0 EFLAGS: 00000246 [ 893.191948] RAX: 0000000000000000 RBX: ffff9832bf380000 RCX: 0000000000710000 [ 893.191949] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 893.191950] RBP: ffff98321b547ab0 R08: ffff9832bf39b8c0 R09: 0000000000000000 [ 893.191950] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321b547a78 [ 893.191951] R13: ffffffffa00ec9b1 R14: ffff98321b547a58 R15: ffff98321b547a88 [ 893.191953] FS: 0000000000000000(0000) GS:ffff9832bf380000(0000) knlGS:0000000000000000 [ 893.191954] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.191955] CR2: 00007ffff7fd6000 CR3: 0000003f67e7a000 CR4: 00000000003607e0 [ 893.191955] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.191956] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.191957] Call Trace: [ 893.191959] [] queued_spin_lock_slowpath+0xb/0xf [ 893.191961] [] _raw_spin_lock+0x30/0x40 [ 893.191967] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 893.191975] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 893.191977] [] ? wake_up_atomic_t+0x30/0x30 [ 893.191985] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.192009] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.192035] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 893.192037] [] ? check_preempt_curr+0x80/0xa0 [ 893.192062] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 893.192085] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 893.192115] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 893.192143] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.192146] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.192173] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.192197] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.192199] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.192200] [] ? sched_feat_set+0xf0/0xf0 [ 893.192225] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.192249] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.192251] [] kthread+0xd1/0xe0 [ 893.192253] [] ? insert_kthread_work+0x40/0x40 [ 893.192255] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.192256] [] ? insert_kthread_work+0x40/0x40 [ 893.192257] Code: [ 893.192258] 13 [ 893.192258] 48 [ 893.192258] c1 [ 893.192259] ea [ 893.192259] 0d [ 893.192259] 48 [ 893.192260] 98 [ 893.192260] 83 [ 893.192260] e2 [ 893.192261] 30 [ 893.192261] 48 [ 893.192261] 81 [ 893.192262] c2 [ 893.192262] c0 [ 893.192262] b8 [ 893.192263] 01 [ 893.192263] 00 [ 893.192264] 48 [ 893.192264] 03 [ 893.192264] 14 [ 893.192265] c5 [ 893.192265] e0 [ 893.192265] 17 [ 893.192266] d5 [ 893.192266] a0 [ 893.192266] 4c [ 893.192267] 89 [ 893.192267] 02 [ 893.192268] 41 [ 893.192268] 8b [ 893.192268] 40 [ 893.192269] 08 [ 893.192269] 85 [ 893.192269] c0 [ 893.192270] 75 [ 893.192270] 0f [ 893.192270] 0f [ 893.192271] 1f [ 893.192271] 44 [ 893.192272] 00 [ 893.192272] 00 [ 893.192272] f3 [ 893.192273] 90 [ 893.192273] <41> [ 893.192273] 8b [ 893.192274] 40 [ 893.192274] 08 [ 893.192275] 85 [ 893.192275] c0 [ 893.192275] 74 [ 893.192276] f6 [ 893.192276] 4d [ 893.192276] 8b [ 893.192277] 08 [ 893.192277] 4d [ 893.192277] 85 [ 893.192278] c9 [ 893.192278] 74 [ 893.192278] 04 [ 893.192279] 41 [ 893.192279] 0f [ 893.192279] 18 [ 893.192280] 09 [ 893.192280] 8b [ 893.192280] [ 893.195880] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [mdt01_007:17310] [ 893.195881] Modules linked in: [ 893.195881] osp(OE) [ 893.195882] mdd(OE) [ 893.195882] lod(OE) [ 893.195883] mdt(OE) [ 893.195883] lfsck(OE) [ 893.195883] mgs(OE) [ 893.195884] mgc(OE) [ 893.195884] osd_zfs(OE) [ 893.195884] lquota(OE) [ 893.195885] fid(OE) [ 893.195885] fld(OE) [ 893.195885] ptlrpc(OE) [ 893.195885] obdclass(OE) [ 893.195886] ko2iblnd(OE) [ 893.195886] lnet(OE) [ 893.195886] libcfs(OE) [ 893.195887] iTCO_wdt [ 893.195887] iTCO_vendor_support [ 893.195887] sb_edac [ 893.195887] intel_powerclamp [ 893.195888] coretemp [ 893.195888] intel_rapl [ 893.195888] iosf_mbi [ 893.195889] rpcrdma [ 893.195889] ib_iser [ 893.195889] kvm [ 893.195889] irqbypass [ 893.195890] joydev [ 893.195890] pcspkr [ 893.195890] lpc_ich [ 893.195890] i2c_i801 [ 893.195891] ioatdma [ 893.195891] ses [ 893.195891] enclosure [ 893.195891] sg [ 893.195892] ipmi_si [ 893.195892] ipmi_devintf [ 893.195892] ipmi_msghandler [ 893.195892] acpi_power_meter [ 893.195893] dm_round_robin [ 893.195893] acpi_cpufreq [ 893.195893] ib_ipoib [ 893.195893] rdma_ucm [ 893.195894] ib_umad [ 893.195894] sch_fq_codel [ 893.195894] iw_cxgb4 [ 893.195894] rdma_cm [ 893.195895] iw_cm [ 893.195895] ib_cm [ 893.195895] iw_cxgb3 [ 893.195896] zfs(POE) [ 893.195896] binfmt_misc [ 893.195896] zunicode(POE) [ 893.195896] zavl(POE) [ 893.195897] icp(POE) [ 893.195897] zcommon(POE) [ 893.195897] znvpair(POE) [ 893.195898] spl(OE) [ 893.195898] msr_safe(OE) [ 893.195898] ip_tables [ 893.195899] nfsv3 [ 893.195899] nfs_acl [ 893.195899] rpcsec_gss_krb5 [ 893.195900] auth_rpcgss [ 893.195900] nfsv4 [ 893.195900] dns_resolver [ 893.195900] nfs [ 893.195901] lockd [ 893.195901] grace [ 893.195901] fscache [ 893.195902] overlay(T) [ 893.195902] ext4 [ 893.195902] mbcache [ 893.195903] jbd2 [ 893.195903] dm_service_time [ 893.195903] sd_mod [ 893.195903] crc_t10dif [ 893.195904] crct10dif_generic [ 893.195904] mlx5_ib [ 893.195904] ib_uverbs [ 893.195904] ib_core [ 893.195905] be2iscsi [ 893.195905] bnx2i [ 893.195906] cnic [ 893.195906] uio [ 893.195906] cxgb4i [ 893.195907] cxgb4 [ 893.195907] cxgb3i [ 893.195907] cxgb3 [ 893.195907] mdio [ 893.195908] libcxgbi [ 893.195908] libcxgb [ 893.195908] qla4xxx [ 893.195908] iscsi_boot_sysfs [ 893.195909] mgag200 [ 893.195909] i2c_algo_bit [ 893.195909] 8021q [ 893.195909] drm_kms_helper [ 893.195910] garp [ 893.195910] syscopyarea [ 893.195910] mrp [ 893.195910] crct10dif_pclmul [ 893.195911] stp [ 893.195911] sysfillrect [ 893.195911] crct10dif_common [ 893.195912] sysimgblt [ 893.195912] crc32_pclmul [ 893.195912] llc [ 893.195912] fb_sys_fops [ 893.195913] crc32c_intel [ 893.195913] ttm [ 893.195913] ghash_clmulni_intel [ 893.195913] mlx5_core [ 893.195914] drm [ 893.195914] mxm_wmi [ 893.195914] ahci [ 893.195915] ixgbe(OE) [ 893.195915] aesni_intel [ 893.195915] mpt3sas [ 893.195915] mlxfw [ 893.195916] dm_multipath [ 893.195916] lrw [ 893.195916] devlink [ 893.195917] libahci [ 893.195917] gf128mul [ 893.195917] dca [ 893.195918] glue_helper [ 893.195918] ablk_helper [ 893.195918] raid_class [ 893.195918] ptp [ 893.195919] drm_panel_orientation_quirks [ 893.195919] libata [ 893.195919] cryptd [ 893.195919] scsi_transport_sas [ 893.195920] pps_core [ 893.195920] wmi [ 893.195920] sunrpc [ 893.195921] dm_mirror [ 893.195921] dm_region_hash [ 893.195921] dm_log [ 893.195921] dm_mod [ 893.195922] iscsi_tcp [ 893.195922] libiscsi_tcp [ 893.195922] libiscsi [ 893.195923] scsi_transport_iscsi [ 893.195923] fuse [ 893.195923] [ 893.195925] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.195925] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.195926] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 893.195927] RIP: 0010:[] [ 893.195929] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 893.195930] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 893.195930] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 893.195931] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 893.195931] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 893.195932] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 893.195933] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 893.195934] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 893.195934] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.195935] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 893.195936] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.195936] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.195937] Call Trace: [ 893.195939] [] queued_spin_lock_slowpath+0xb/0xf [ 893.195941] [] _raw_spin_lock+0x30/0x40 [ 893.195946] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 893.195952] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 893.195954] [] ? wake_up_atomic_t+0x30/0x30 [ 893.195960] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.195978] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.195996] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 893.195998] [] ? check_preempt_curr+0x80/0xa0 [ 893.196015] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 893.196032] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 893.196055] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 893.196077] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.196078] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.196098] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.196116] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.196117] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.196118] [] ? sched_feat_set+0xf0/0xf0 [ 893.196137] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.196155] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.196156] [] kthread+0xd1/0xe0 [ 893.196158] [] ? insert_kthread_work+0x40/0x40 [ 893.196159] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.196161] [] ? insert_kthread_work+0x40/0x40 [ 893.196161] Code: [ 893.196162] 0d [ 893.196162] 48 [ 893.196162] 98 [ 893.196162] 83 [ 893.196163] e2 [ 893.196163] 30 [ 893.196163] 48 [ 893.196163] 81 [ 893.196164] c2 [ 893.196164] c0 [ 893.196164] b8 [ 893.196164] 01 [ 893.196165] 00 [ 893.196165] 48 [ 893.196165] 03 [ 893.196165] 14 [ 893.196166] c5 [ 893.196166] e0 [ 893.196166] 17 [ 893.196166] d5 [ 893.196167] a0 [ 893.196167] 4c [ 893.196167] 89 [ 893.196168] 02 [ 893.196168] 41 [ 893.196168] 8b [ 893.196168] 40 [ 893.196169] 08 [ 893.196169] 85 [ 893.196169] c0 [ 893.196169] 75 [ 893.196170] 0f [ 893.196170] 0f [ 893.196170] 1f [ 893.196171] 44 [ 893.196171] 00 [ 893.196171] 00 [ 893.196171] f3 [ 893.196172] 90 [ 893.196172] 41 [ 893.196172] 8b [ 893.196172] 40 [ 893.196173] 08 [ 893.196173] <85> [ 893.196173] c0 [ 893.196173] 74 [ 893.196174] f6 [ 893.196174] 4d [ 893.196174] 8b [ 893.196174] 08 [ 893.196175] 4d [ 893.196175] 85 [ 893.196175] c9 [ 893.196175] 74 [ 893.196176] 04 [ 893.196176] 41 [ 893.196176] 0f [ 893.196176] 18 [ 893.196177] 09 [ 893.196177] 8b [ 893.196177] 17 [ 893.196177] 0f [ 893.196177] b7 [ 893.196178] c2 [ 893.196178] [ 893.209882] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_mgs_0013:10822] [ 893.209882] Modules linked in: [ 893.209883] osp(OE) [ 893.209884] mdd(OE) [ 893.209884] lod(OE) [ 893.209885] mdt(OE) [ 893.209885] lfsck(OE) [ 893.209885] mgs(OE) [ 893.209886] mgc(OE) [ 893.209886] osd_zfs(OE) [ 893.209886] lquota(OE) [ 893.209886] fid(OE) [ 893.209887] fld(OE) [ 893.209887] ptlrpc(OE) [ 893.209887] obdclass(OE) [ 893.209888] ko2iblnd(OE) [ 893.209888] lnet(OE) [ 893.209888] libcfs(OE) [ 893.209889] iTCO_wdt [ 893.209889] iTCO_vendor_support [ 893.209889] sb_edac [ 893.209889] intel_powerclamp [ 893.209890] coretemp [ 893.209890] intel_rapl [ 893.209890] iosf_mbi [ 893.209890] rpcrdma [ 893.209891] ib_iser [ 893.209891] kvm [ 893.209891] irqbypass [ 893.209892] joydev [ 893.209892] pcspkr [ 893.209892] lpc_ich [ 893.209892] i2c_i801 [ 893.209893] ioatdma [ 893.209893] ses [ 893.209893] enclosure [ 893.209893] sg [ 893.209894] ipmi_si [ 893.209894] ipmi_devintf [ 893.209894] ipmi_msghandler [ 893.209895] acpi_power_meter [ 893.209895] dm_round_robin [ 893.209895] acpi_cpufreq [ 893.209896] ib_ipoib [ 893.209896] rdma_ucm [ 893.209896] ib_umad [ 893.209896] sch_fq_codel [ 893.209897] iw_cxgb4 [ 893.209897] rdma_cm [ 893.209897] iw_cm [ 893.209898] ib_cm [ 893.209898] iw_cxgb3 [ 893.209898] zfs(POE) [ 893.209899] binfmt_misc [ 893.209899] zunicode(POE) [ 893.209899] zavl(POE) [ 893.209900] icp(POE) [ 893.209900] zcommon(POE) [ 893.209900] znvpair(POE) [ 893.209900] spl(OE) [ 893.209901] msr_safe(OE) [ 893.209901] ip_tables [ 893.209901] nfsv3 [ 893.209902] nfs_acl [ 893.209902] rpcsec_gss_krb5 [ 893.209902] auth_rpcgss [ 893.209903] nfsv4 [ 893.209903] dns_resolver [ 893.209904] nfs [ 893.209904] lockd [ 893.209904] grace [ 893.209904] fscache [ 893.209905] overlay(T) [ 893.209905] ext4 [ 893.209905] mbcache [ 893.209906] jbd2 [ 893.209906] dm_service_time [ 893.209906] sd_mod [ 893.209907] crc_t10dif [ 893.209907] crct10dif_generic [ 893.209907] mlx5_ib [ 893.209908] ib_uverbs [ 893.209908] ib_core [ 893.209908] be2iscsi [ 893.209909] bnx2i [ 893.209909] cnic [ 893.209909] uio [ 893.209909] cxgb4i [ 893.209910] cxgb4 [ 893.209910] cxgb3i [ 893.209910] cxgb3 [ 893.209910] mdio [ 893.209911] libcxgbi [ 893.209911] libcxgb [ 893.209911] qla4xxx [ 893.209912] iscsi_boot_sysfs [ 893.209912] mgag200 [ 893.209912] i2c_algo_bit [ 893.209913] 8021q [ 893.209913] drm_kms_helper [ 893.209914] garp [ 893.209915] syscopyarea [ 893.209915] mrp [ 893.209916] crct10dif_pclmul [ 893.209916] stp [ 893.209916] sysfillrect [ 893.209917] crct10dif_common [ 893.209917] sysimgblt [ 893.209917] crc32_pclmul [ 893.209918] llc [ 893.209918] fb_sys_fops [ 893.209919] crc32c_intel [ 893.209919] ttm [ 893.209920] ghash_clmulni_intel [ 893.209920] mlx5_core [ 893.209920] drm [ 893.209921] mxm_wmi [ 893.209921] ahci [ 893.209922] ixgbe(OE) [ 893.209922] aesni_intel [ 893.209923] mpt3sas [ 893.209923] mlxfw [ 893.209924] dm_multipath [ 893.209924] lrw [ 893.209925] devlink [ 893.209925] libahci [ 893.209926] gf128mul [ 893.209926] dca [ 893.209926] glue_helper [ 893.209927] ablk_helper [ 893.209928] raid_class [ 893.209928] ptp [ 893.209929] drm_panel_orientation_quirks [ 893.209929] libata [ 893.209930] cryptd [ 893.209930] scsi_transport_sas [ 893.209931] pps_core [ 893.209931] wmi [ 893.209932] sunrpc [ 893.209932] dm_mirror [ 893.209933] dm_region_hash [ 893.209933] dm_log [ 893.209934] dm_mod [ 893.209934] iscsi_tcp [ 893.209935] libiscsi_tcp [ 893.209935] libiscsi [ 893.209936] scsi_transport_iscsi [ 893.209936] fuse [ 893.209936] [ 893.209938] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.209939] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.209940] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 893.209940] RIP: 0010:[] [ 893.209945] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 893.209946] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 893.209946] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 893.209947] RDX: ffff9832bf55b8c0 RSI: 0000000000e90000 RDI: ffff97f2a83bcf40 [ 893.209947] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 893.209948] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 893.209948] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 893.209950] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 893.209950] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.209951] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 893.209952] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.209953] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.209953] Call Trace: [ 893.209958] [] queued_spin_lock_slowpath+0xb/0xf [ 893.209961] [] _raw_spin_lock+0x30/0x40 [ 893.209975] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 893.209979] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 893.209991] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 893.209994] [] ? wake_up_atomic_t+0x30/0x30 [ 893.209999] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.210041] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.210058] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 893.210060] [] ? enqueue_task_fair+0x208/0x6c0 [ 893.210062] [] ? check_preempt_curr+0x80/0xa0 [ 893.210063] [] ? ttwu_do_wakeup+0x19/0x100 [ 893.210098] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 893.210125] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.210127] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.210152] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.210178] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.210179] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.210180] [] ? sched_feat_set+0xf0/0xf0 [ 893.210206] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.210208] [] ? __switch_to+0xce/0x5a0 [ 893.210234] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.210236] [] kthread+0xd1/0xe0 [ 893.210237] [] ? insert_kthread_work+0x40/0x40 [ 893.210239] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.210241] [] ? insert_kthread_work+0x40/0x40 [ 893.210241] Code: [ 893.210242] 13 [ 893.210242] 48 [ 893.210242] c1 [ 893.210242] ea [ 893.210243] 0d [ 893.210243] 48 [ 893.210243] 98 [ 893.210244] 83 [ 893.210244] e2 [ 893.210244] 30 [ 893.210244] 48 [ 893.210245] 81 [ 893.210245] c2 [ 893.210245] c0 [ 893.210245] b8 [ 893.210246] 01 [ 893.210246] 00 [ 893.210246] 48 [ 893.210246] 03 [ 893.210247] 14 [ 893.210247] c5 [ 893.210248] e0 [ 893.210248] 17 [ 893.210248] d5 [ 893.210249] a0 [ 893.210249] 4c [ 893.210249] 89 [ 893.210249] 02 [ 893.210250] 41 [ 893.210250] 8b [ 893.210250] 40 [ 893.210250] 08 [ 893.210251] 85 [ 893.210251] c0 [ 893.210251] 75 [ 893.210251] 0f [ 893.210252] 0f [ 893.210252] 1f [ 893.210252] 44 [ 893.210252] 00 [ 893.210253] 00 [ 893.210253] f3 [ 893.210253] 90 [ 893.210254] <41> [ 893.210254] 8b [ 893.210254] 40 [ 893.210254] 08 [ 893.210255] 85 [ 893.210255] c0 [ 893.210255] 74 [ 893.210255] f6 [ 893.210256] 4d [ 893.210256] 8b [ 893.210256] 08 [ 893.210256] 4d [ 893.210256] 85 [ 893.210257] c9 [ 893.210257] 74 [ 893.210257] 04 [ 893.210257] 41 [ 893.210258] 0f [ 893.210258] 18 [ 893.210258] 09 [ 893.210258] 8b [ 893.210259] [ 893.215882] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_mgs_0014:10826] [ 893.215883] Modules linked in: [ 893.215884] osp(OE) [ 893.215884] mdd(OE) [ 893.215885] lod(OE) [ 893.215885] mdt(OE) [ 893.215885] lfsck(OE) [ 893.215886] mgs(OE) [ 893.215886] mgc(OE) [ 893.215886] osd_zfs(OE) [ 893.215886] lquota(OE) [ 893.215887] fid(OE) [ 893.215887] fld(OE) [ 893.215887] ptlrpc(OE) [ 893.215888] obdclass(OE) [ 893.215888] ko2iblnd(OE) [ 893.215888] lnet(OE) [ 893.215889] libcfs(OE) [ 893.215889] iTCO_wdt [ 893.215889] iTCO_vendor_support [ 893.215890] sb_edac [ 893.215890] intel_powerclamp [ 893.215890] coretemp [ 893.215891] intel_rapl [ 893.215891] iosf_mbi [ 893.215891] rpcrdma [ 893.215891] ib_iser [ 893.215892] kvm [ 893.215892] irqbypass [ 893.215892] joydev [ 893.215893] pcspkr [ 893.215893] lpc_ich [ 893.215893] i2c_i801 [ 893.215893] ioatdma [ 893.215894] ses [ 893.215894] enclosure [ 893.215894] sg [ 893.215894] ipmi_si [ 893.215895] ipmi_devintf [ 893.215895] ipmi_msghandler [ 893.215895] acpi_power_meter [ 893.215896] dm_round_robin [ 893.215896] acpi_cpufreq [ 893.215896] ib_ipoib [ 893.215896] rdma_ucm [ 893.215897] ib_umad [ 893.215897] sch_fq_codel [ 893.215897] iw_cxgb4 [ 893.215897] rdma_cm [ 893.215898] iw_cm [ 893.215898] ib_cm [ 893.215898] iw_cxgb3 [ 893.215899] zfs(POE) [ 893.215899] binfmt_misc [ 893.215899] zunicode(POE) [ 893.215900] zavl(POE) [ 893.215900] icp(POE) [ 893.215900] zcommon(POE) [ 893.215901] znvpair(POE) [ 893.215901] spl(OE) [ 893.215901] msr_safe(OE) [ 893.215901] ip_tables [ 893.215902] nfsv3 [ 893.215902] nfs_acl [ 893.215902] rpcsec_gss_krb5 [ 893.215903] auth_rpcgss [ 893.215903] nfsv4 [ 893.215903] dns_resolver [ 893.215904] nfs [ 893.215904] lockd [ 893.215904] grace [ 893.215904] fscache [ 893.215905] overlay(T) [ 893.215905] ext4 [ 893.215905] mbcache [ 893.215906] jbd2 [ 893.215906] dm_service_time [ 893.215906] sd_mod [ 893.215906] crc_t10dif [ 893.215907] crct10dif_generic [ 893.215907] mlx5_ib [ 893.215907] ib_uverbs [ 893.215908] ib_core [ 893.215908] be2iscsi [ 893.215908] bnx2i [ 893.215908] cnic [ 893.215909] uio [ 893.215909] cxgb4i [ 893.215909] cxgb4 [ 893.215909] cxgb3i [ 893.215910] cxgb3 [ 893.215910] mdio [ 893.215910] libcxgbi [ 893.215911] libcxgb [ 893.215911] qla4xxx [ 893.215911] iscsi_boot_sysfs [ 893.215911] mgag200 [ 893.215912] i2c_algo_bit [ 893.215912] 8021q [ 893.215912] drm_kms_helper [ 893.215912] garp [ 893.215913] syscopyarea [ 893.215913] mrp [ 893.215913] crct10dif_pclmul [ 893.215913] stp [ 893.215914] sysfillrect [ 893.215914] crct10dif_common [ 893.215914] sysimgblt [ 893.215914] crc32_pclmul [ 893.215915] llc [ 893.215915] fb_sys_fops [ 893.215915] crc32c_intel [ 893.215915] ttm [ 893.215916] ghash_clmulni_intel [ 893.215916] mlx5_core [ 893.215916] drm [ 893.215917] mxm_wmi [ 893.215917] ahci [ 893.215917] ixgbe(OE) [ 893.215917] aesni_intel [ 893.215918] mpt3sas [ 893.215918] mlxfw [ 893.215918] dm_multipath [ 893.215919] lrw [ 893.215919] devlink [ 893.215919] libahci [ 893.215920] gf128mul [ 893.215920] dca [ 893.215920] glue_helper [ 893.215921] ablk_helper [ 893.215921] raid_class [ 893.215921] ptp [ 893.215921] drm_panel_orientation_quirks [ 893.215922] libata [ 893.215922] cryptd [ 893.215922] scsi_transport_sas [ 893.215923] pps_core [ 893.215923] wmi [ 893.215923] sunrpc [ 893.215923] dm_mirror [ 893.215924] dm_region_hash [ 893.215924] dm_log [ 893.215924] dm_mod [ 893.215924] iscsi_tcp [ 893.215925] libiscsi_tcp [ 893.215925] libiscsi [ 893.215925] scsi_transport_iscsi [ 893.215926] fuse [ 893.215926] [ 893.215927] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.215928] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.215928] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 893.215929] RIP: 0010:[] [ 893.215931] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 893.215932] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 893.215932] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 893.215933] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 893.215934] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 893.215934] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 893.215935] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 893.215936] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 893.215937] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.215937] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 893.215938] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.215939] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.215939] Call Trace: [ 893.215941] [] queued_spin_lock_slowpath+0xb/0xf [ 893.215943] [] _raw_spin_lock+0x30/0x40 [ 893.215948] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 893.215952] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 893.215958] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 893.215960] [] ? wake_up_atomic_t+0x30/0x30 [ 893.215966] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.215989] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.216014] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 893.216015] [] ? check_preempt_curr+0x80/0xa0 [ 893.216039] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 893.216063] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 893.216091] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 893.216117] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.216119] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.216144] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.216169] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.216171] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.216172] [] ? sched_feat_set+0xf0/0xf0 [ 893.216197] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.216199] [] ? __switch_to+0xce/0x5a0 [ 893.216224] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.216226] [] kthread+0xd1/0xe0 [ 893.216227] [] ? insert_kthread_work+0x40/0x40 [ 893.216229] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.216230] [] ? insert_kthread_work+0x40/0x40 [ 893.216231] Code: [ 893.216231] 0d [ 893.216231] 48 [ 893.216232] 98 [ 893.216232] 83 [ 893.216232] e2 [ 893.216232] 30 [ 893.216233] 48 [ 893.216233] 81 [ 893.216233] c2 [ 893.216233] c0 [ 893.216234] b8 [ 893.216234] 01 [ 893.216234] 00 [ 893.216235] 48 [ 893.216235] 03 [ 893.216235] 14 [ 893.216235] c5 [ 893.216236] e0 [ 893.216236] 17 [ 893.216236] d5 [ 893.216236] a0 [ 893.216237] 4c [ 893.216237] 89 [ 893.216237] 02 [ 893.216238] 41 [ 893.216238] 8b [ 893.216238] 40 [ 893.216238] 08 [ 893.216239] 85 [ 893.216239] c0 [ 893.216239] 75 [ 893.216239] 0f [ 893.216240] 0f [ 893.216240] 1f [ 893.216240] 44 [ 893.216240] 00 [ 893.216241] 00 [ 893.216241] f3 [ 893.216241] 90 [ 893.216241] 41 [ 893.216242] 8b [ 893.216242] 40 [ 893.216242] 08 [ 893.216243] <85> [ 893.216243] c0 [ 893.216243] 74 [ 893.216243] f6 [ 893.216243] 4d [ 893.216244] 8b [ 893.216244] 08 [ 893.216244] 4d [ 893.216244] 85 [ 893.216245] c9 [ 893.216245] 74 [ 893.216245] 04 [ 893.216245] 41 [ 893.216246] 0f [ 893.216246] 18 [ 893.216246] 09 [ 893.216246] 8b [ 893.216247] 17 [ 893.216247] 0f [ 893.216247] b7 [ 893.216248] c2 [ 893.216248] [ 893.227884] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 22s! [mdt01_014:17318] [ 893.227885] Modules linked in: [ 893.227885] osp(OE) [ 893.227886] mdd(OE) [ 893.227886] lod(OE) [ 893.227887] mdt(OE) [ 893.227887] lfsck(OE) [ 893.227887] mgs(OE) [ 893.227887] mgc(OE) [ 893.227888] osd_zfs(OE) [ 893.227888] lquota(OE) [ 893.227888] fid(OE) [ 893.227889] fld(OE) [ 893.227889] ptlrpc(OE) [ 893.227889] obdclass(OE) [ 893.227890] ko2iblnd(OE) [ 893.227890] lnet(OE) [ 893.227890] libcfs(OE) [ 893.227890] iTCO_wdt [ 893.227891] iTCO_vendor_support [ 893.227891] sb_edac [ 893.227891] intel_powerclamp [ 893.227892] coretemp [ 893.227892] intel_rapl [ 893.227892] iosf_mbi [ 893.227892] rpcrdma [ 893.227893] ib_iser [ 893.227893] kvm [ 893.227893] irqbypass [ 893.227893] joydev [ 893.227894] pcspkr [ 893.227894] lpc_ich [ 893.227894] i2c_i801 [ 893.227894] ioatdma [ 893.227895] ses [ 893.227895] enclosure [ 893.227895] sg [ 893.227895] ipmi_si [ 893.227896] ipmi_devintf [ 893.227896] ipmi_msghandler [ 893.227896] acpi_power_meter [ 893.227897] dm_round_robin [ 893.227897] acpi_cpufreq [ 893.227897] ib_ipoib [ 893.227897] rdma_ucm [ 893.227898] ib_umad [ 893.227898] sch_fq_codel [ 893.227898] iw_cxgb4 [ 893.227899] rdma_cm [ 893.227899] iw_cm [ 893.227899] ib_cm [ 893.227900] iw_cxgb3 [ 893.227900] zfs(POE) [ 893.227901] binfmt_misc [ 893.227901] zunicode(POE) [ 893.227901] zavl(POE) [ 893.227902] icp(POE) [ 893.227902] zcommon(POE) [ 893.227903] znvpair(POE) [ 893.227903] spl(OE) [ 893.227903] msr_safe(OE) [ 893.227904] ip_tables [ 893.227904] nfsv3 [ 893.227904] nfs_acl [ 893.227905] rpcsec_gss_krb5 [ 893.227905] auth_rpcgss [ 893.227905] nfsv4 [ 893.227905] dns_resolver [ 893.227906] nfs [ 893.227906] lockd [ 893.227906] grace [ 893.227907] fscache [ 893.227907] overlay(T) [ 893.227907] ext4 [ 893.227907] mbcache [ 893.227908] jbd2 [ 893.227908] dm_service_time [ 893.227908] sd_mod [ 893.227908] crc_t10dif [ 893.227909] crct10dif_generic [ 893.227909] mlx5_ib [ 893.227909] ib_uverbs [ 893.227910] ib_core [ 893.227910] be2iscsi [ 893.227910] bnx2i [ 893.227910] cnic [ 893.227911] uio [ 893.227911] cxgb4i [ 893.227911] cxgb4 [ 893.227911] cxgb3i [ 893.227912] cxgb3 [ 893.227912] mdio [ 893.227912] libcxgbi [ 893.227912] libcxgb [ 893.227913] qla4xxx [ 893.227913] iscsi_boot_sysfs [ 893.227913] mgag200 [ 893.227913] i2c_algo_bit [ 893.227914] 8021q [ 893.227914] drm_kms_helper [ 893.227914] garp [ 893.227915] syscopyarea [ 893.227915] mrp [ 893.227915] crct10dif_pclmul [ 893.227915] stp [ 893.227916] sysfillrect [ 893.227916] crct10dif_common [ 893.227916] sysimgblt [ 893.227916] crc32_pclmul [ 893.227917] llc [ 893.227917] fb_sys_fops [ 893.227917] crc32c_intel [ 893.227917] ttm [ 893.227918] ghash_clmulni_intel [ 893.227918] mlx5_core [ 893.227919] drm [ 893.227919] mxm_wmi [ 893.227919] ahci [ 893.227920] ixgbe(OE) [ 893.227920] aesni_intel [ 893.227920] mpt3sas [ 893.227920] mlxfw [ 893.227921] dm_multipath [ 893.227921] lrw [ 893.227921] devlink [ 893.227922] libahci [ 893.227922] gf128mul [ 893.227922] dca [ 893.227922] glue_helper [ 893.227923] ablk_helper [ 893.227923] raid_class [ 893.227923] ptp [ 893.227924] drm_panel_orientation_quirks [ 893.227924] libata [ 893.227924] cryptd [ 893.227925] scsi_transport_sas [ 893.227925] pps_core [ 893.227925] wmi [ 893.227926] sunrpc [ 893.227926] dm_mirror [ 893.227926] dm_region_hash [ 893.227926] dm_log [ 893.227927] dm_mod [ 893.227927] iscsi_tcp [ 893.227927] libiscsi_tcp [ 893.227927] libiscsi [ 893.227928] scsi_transport_iscsi [ 893.227928] fuse [ 893.227928] [ 893.227930] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.227931] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.227931] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 893.227932] RIP: 0010:[] [ 893.227934] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 893.227935] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 893.227935] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 893.227936] RDX: ffff9832bf21b8c0 RSI: 0000000000410000 RDI: ffff97f2a83bcf40 [ 893.227937] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 893.227937] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 893.227938] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 893.227939] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 893.227940] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.227941] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 893.227942] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.227942] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.227943] Call Trace: [ 893.227945] [] queued_spin_lock_slowpath+0xb/0xf [ 893.227946] [] _raw_spin_lock+0x30/0x40 [ 893.227951] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 893.227958] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 893.227960] [] ? wake_up_atomic_t+0x30/0x30 [ 893.227965] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.227984] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.228002] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 893.228021] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 893.228039] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 893.228056] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 893.228079] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 893.228100] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.228101] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.228121] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.228139] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.228140] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.228141] [] ? sched_feat_set+0xf0/0xf0 [ 893.228160] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.228178] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.228180] [] kthread+0xd1/0xe0 [ 893.228181] [] ? insert_kthread_work+0x40/0x40 [ 893.228182] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.228184] [] ? insert_kthread_work+0x40/0x40 [ 893.228184] Code: [ 893.228185] 0d [ 893.228185] 48 [ 893.228185] 98 [ 893.228185] 83 [ 893.228186] e2 [ 893.228186] 30 [ 893.228186] 48 [ 893.228186] 81 [ 893.228187] c2 [ 893.228187] c0 [ 893.228187] b8 [ 893.228188] 01 [ 893.228188] 00 [ 893.228188] 48 [ 893.228188] 03 [ 893.228189] 14 [ 893.228189] c5 [ 893.228189] e0 [ 893.228190] 17 [ 893.228190] d5 [ 893.228190] a0 [ 893.228190] 4c [ 893.228190] 89 [ 893.228191] 02 [ 893.228191] 41 [ 893.228191] 8b [ 893.228192] 40 [ 893.228192] 08 [ 893.228192] 85 [ 893.228192] c0 [ 893.228193] 75 [ 893.228193] 0f [ 893.228193] 0f [ 893.228193] 1f [ 893.228194] 44 [ 893.228194] 00 [ 893.228194] 00 [ 893.228195] f3 [ 893.228195] 90 [ 893.228195] 41 [ 893.228195] 8b [ 893.228196] 40 [ 893.228196] 08 [ 893.228196] <85> [ 893.228196] c0 [ 893.228197] 74 [ 893.228197] f6 [ 893.228197] 4d [ 893.228197] 8b [ 893.228198] 08 [ 893.228198] 4d [ 893.228198] 85 [ 893.228198] c9 [ 893.228199] 74 [ 893.228199] 04 [ 893.228199] 41 [ 893.228199] 0f [ 893.228200] 18 [ 893.228200] 09 [ 893.228200] 8b [ 893.228200] 17 [ 893.228201] 0f [ 893.228201] b7 [ 893.228201] c2 [ 893.228201] [ 893.232884] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [mdt01_017:17555] [ 893.232885] Modules linked in: [ 893.232885] osp(OE) [ 893.232886] mdd(OE) [ 893.232886] lod(OE) [ 893.232886] mdt(OE) [ 893.232887] lfsck(OE) [ 893.232887] mgs(OE) [ 893.232888] mgc(OE) [ 893.232888] osd_zfs(OE) [ 893.232889] lquota(OE) [ 893.232889] fid(OE) [ 893.232889] fld(OE) [ 893.232890] ptlrpc(OE) [ 893.232890] obdclass(OE) [ 893.232891] ko2iblnd(OE) [ 893.232892] lnet(OE) [ 893.232892] libcfs(OE) [ 893.232893] iTCO_wdt [ 893.232893] iTCO_vendor_support [ 893.232893] sb_edac [ 893.232894] intel_powerclamp [ 893.232894] coretemp [ 893.232895] intel_rapl [ 893.232895] iosf_mbi [ 893.232896] rpcrdma [ 893.232896] ib_iser [ 893.232896] kvm [ 893.232897] irqbypass [ 893.232897] joydev [ 893.232898] pcspkr [ 893.232898] lpc_ich [ 893.232898] i2c_i801 [ 893.232899] ioatdma [ 893.232899] ses [ 893.232900] enclosure [ 893.232900] sg [ 893.232901] ipmi_si [ 893.232901] ipmi_devintf [ 893.232902] ipmi_msghandler [ 893.232902] acpi_power_meter [ 893.232902] dm_round_robin [ 893.232903] acpi_cpufreq [ 893.232903] ib_ipoib [ 893.232904] rdma_ucm [ 893.232904] ib_umad [ 893.232904] sch_fq_codel [ 893.232905] iw_cxgb4 [ 893.232905] rdma_cm [ 893.232906] iw_cm [ 893.232906] ib_cm [ 893.232906] iw_cxgb3 [ 893.232907] zfs(POE) [ 893.232907] binfmt_misc [ 893.232908] zunicode(POE) [ 893.232908] zavl(POE) [ 893.232909] icp(POE) [ 893.232909] zcommon(POE) [ 893.232910] znvpair(POE) [ 893.232910] spl(OE) [ 893.232911] msr_safe(OE) [ 893.232911] ip_tables [ 893.232912] nfsv3 [ 893.232912] nfs_acl [ 893.232913] rpcsec_gss_krb5 [ 893.232913] auth_rpcgss [ 893.232913] nfsv4 [ 893.232914] dns_resolver [ 893.232914] nfs [ 893.232915] lockd [ 893.232915] grace [ 893.232915] fscache [ 893.232916] overlay(T) [ 893.232916] ext4 [ 893.232917] mbcache [ 893.232917] jbd2 [ 893.232918] dm_service_time [ 893.232918] sd_mod [ 893.232918] crc_t10dif [ 893.232919] crct10dif_generic [ 893.232919] mlx5_ib [ 893.232920] ib_uverbs [ 893.232920] ib_core [ 893.232921] be2iscsi [ 893.232921] bnx2i [ 893.232922] cnic [ 893.232922] uio [ 893.232923] cxgb4i [ 893.232923] cxgb4 [ 893.232923] cxgb3i [ 893.232924] cxgb3 [ 893.232924] mdio [ 893.232925] libcxgbi [ 893.232925] libcxgb [ 893.232926] qla4xxx [ 893.232926] iscsi_boot_sysfs [ 893.232926] mgag200 [ 893.232927] i2c_algo_bit [ 893.232927] 8021q [ 893.232928] drm_kms_helper [ 893.232928] garp [ 893.232928] syscopyarea [ 893.232929] mrp [ 893.232929] crct10dif_pclmul [ 893.232929] stp [ 893.232930] sysfillrect [ 893.232930] crct10dif_common [ 893.232931] sysimgblt [ 893.232931] crc32_pclmul [ 893.232931] llc [ 893.232932] fb_sys_fops [ 893.232932] crc32c_intel [ 893.232932] ttm [ 893.232933] ghash_clmulni_intel [ 893.232933] mlx5_core [ 893.232934] drm [ 893.232934] mxm_wmi [ 893.232934] ahci [ 893.232935] ixgbe(OE) [ 893.232935] aesni_intel [ 893.232936] mpt3sas [ 893.232936] mlxfw [ 893.232936] dm_multipath [ 893.232937] lrw [ 893.232937] devlink [ 893.232937] libahci [ 893.232938] gf128mul [ 893.232938] dca [ 893.232939] glue_helper [ 893.232939] ablk_helper [ 893.232939] raid_class [ 893.232940] ptp [ 893.232940] drm_panel_orientation_quirks [ 893.232941] libata [ 893.232941] cryptd [ 893.232942] scsi_transport_sas [ 893.232942] pps_core [ 893.232943] wmi [ 893.232943] sunrpc [ 893.232943] dm_mirror [ 893.232944] dm_region_hash [ 893.232944] dm_log [ 893.232945] dm_mod [ 893.232945] iscsi_tcp [ 893.232946] libiscsi_tcp [ 893.232946] libiscsi [ 893.232947] scsi_transport_iscsi [ 893.232947] fuse [ 893.232947] [ 893.232950] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.232950] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.232951] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 893.232952] RIP: 0010:[] [ 893.232954] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 893.232955] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000246 [ 893.232956] RAX: 0000000000000000 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 893.232957] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 893.232957] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: 0000000000000000 [ 893.232958] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 893.232959] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 893.232960] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 893.232961] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.232962] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 893.232963] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.232964] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.232964] Call Trace: [ 893.232967] [] queued_spin_lock_slowpath+0xb/0xf [ 893.232968] [] _raw_spin_lock+0x30/0x40 [ 893.232974] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 893.232983] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 893.232985] [] ? wake_up_atomic_t+0x30/0x30 [ 893.232993] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.233017] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.233042] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 893.233043] [] ? check_preempt_curr+0x80/0xa0 [ 893.233067] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 893.233090] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 893.233120] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 893.233148] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.233150] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.233176] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.233201] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.233262] iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 893.233264] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.233265] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.233266] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 893.233268] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x138/0x200 [ 893.233269] RSP: 0018:ffff98321b877a50 EFLAGS: 00000286 [ 893.233270] RAX: 0000000000000001 RBX: ffff97f2bfa80000 RCX: 0000000000110000 [ 893.233271] RDX: 0000000000a10001 RSI: 0000000000f10000 RDI: ffff97f2a83bcf40 [ 893.233271] RBP: ffff98321b877a50 R08: ffff97f2bfa9b8c0 R09: ffff9832bf3db8c0 [ 893.233272] R10: 0000000000000001 R11: 000000000000242e R12: ffff98321b877a18 [ 893.233272] R13: ffff97f2bfc9ad40 R14: ffffffffa00f0602 R15: ffff98321b877a28 [ 893.233273] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 893.233274] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.233275] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 893.233275] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.233276] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.233276] Call Trace: [ 893.233278] [] queued_spin_lock_slowpath+0xb/0xf [ 893.233280] [] _raw_spin_lock+0x30/0x40 [ 893.233285] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 893.233290] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 893.233296] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 893.233297] [] ? wake_up_atomic_t+0x30/0x30 [ 893.233303] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.233322] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.233337] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 893.233355] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 893.233357] [] ? enqueue_task_fair+0x208/0x6c0 [ 893.233359] [] ? sched_clock_cpu+0xa5/0xe0 [ 893.233360] [] ? check_preempt_curr+0x80/0xa0 [ 893.233361] [] ? ttwu_do_wakeup+0x19/0x100 [ 893.233384] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 893.233405] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.233407] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.233428] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.233446] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.233448] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.233449] [] ? sched_feat_set+0xf0/0xf0 [ 893.233467] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.233469] [] ? __switch_to+0xce/0x5a0 [ 893.233487] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.233488] [] kthread+0xd1/0xe0 [ 893.233490] [] ? insert_kthread_work+0x40/0x40 [ 893.233491] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.233492] [] ? insert_kthread_work+0x40/0x40 [ 893.233506] Code: 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 <0f> b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 [ 893.234885] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 22s! [mdt01_018:17592] [ 893.234913] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 893.234934] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 893.234936] CPU: 30 PID: 17592 Comm: mdt01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 893.234936] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 893.234937] task: ffff983230579080 ti: ffff9832469b0000 task.ti: ffff9832469b0000 [ 893.234940] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 893.234941] RSP: 0018:ffff9832469b3a50 EFLAGS: 00000246 [ 893.234942] RAX: 0000000000000000 RBX: ffff9832bf580000 RCX: 0000000000f10000 [ 893.234942] RDX: ffff97f2bfd1b8c0 RSI: 0000000000a10000 RDI: ffff97f2a83bcf40 [ 893.234943] RBP: ffff9832469b3a50 R08: ffff9832bf59b8c0 R09: 0000000000000000 [ 893.234944] R10: 0000000000000002 R11: ffff983211336600 R12: ffff9832469b3a18 [ 893.234945] R13: ffffffffa00ec9b1 R14: ffff9832469b39f8 R15: ffff9832469b3a28 [ 893.234946] FS: 0000000000000000(0000) GS:ffff9832bf580000(0000) knlGS:0000000000000000 [ 893.234947] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 893.234948] CR2: 00007ffff7fc5010 CR3: 0000003f74514000 CR4: 00000000003607e0 [ 893.234949] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 893.234950] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 893.234950] Call Trace: [ 893.234952] [] queued_spin_lock_slowpath+0xb/0xf [ 893.234954] [] _raw_spin_lock+0x30/0x40 [ 893.234960] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 893.234968] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 893.234970] [] ? wake_up_atomic_t+0x30/0x30 [ 893.234978] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 893.235002] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 893.235025] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 893.235027] [] ? check_preempt_curr+0x80/0xa0 [ 893.235028] [] ? ttwu_do_wakeup+0x19/0x100 [ 893.235059] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 893.235087] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 893.235088] [] ? __getnstimeofday64+0x3f/0xd0 [ 893.235114] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 893.235139] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 893.235140] [] ? __wake_up_common_lock+0x91/0xc0 [ 893.235141] [] ? sched_feat_set+0xf0/0xf0 [ 893.235166] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 893.235190] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 893.235192] [] kthread+0xd1/0xe0 [ 893.235194] [] ? insert_kthread_work+0x40/0x40 [ 893.235195] [] ret_from_fork_nospec_begin+0x21/0x21 [ 893.235197] [] ? insert_kthread_work+0x40/0x40 [ 893.235216] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 897.509095] [] ? __wake_up_common_lock+0x91/0xc0 [ 897.516574] [] ? sched_feat_set+0xf0/0xf0 [ 897.524940] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 897.533970] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 897.544241] [] kthread+0xd1/0xe0 [ 897.550169] [] ? insert_kthread_work+0x40/0x40 [ 897.559001] [] ret_from_fork_nospec_begin+0x21/0x21 [ 897.566774] [] ? insert_kthread_work+0x40/0x40 [ 897.575606] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 903.454330] Lustre: MGS: Connection restored to 78346956-244a-44ce-a9fa-70d9e0444cc9 (at 192.168.128.215@o2ib21) [ 903.466177] Lustre: Skipped 2144 previous similar messages [ 905.580896] Lustre: ls1-MDT0000: Client 8c69758f-204f-2e6c-9677-723499fc69d6 (at 192.168.131.196@o2ib39) reconnecting [ 905.582973] LustreError: 137-5: ls1-MDT0001_UUID: not available for connect from 192.168.130.75@o2ib39 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 905.582974] LustreError: Skipped 12 previous similar messages [ 905.620405] Lustre: Skipped 640 previous similar messages [ 907.981593] LNet: Service thread pid 17553 was inactive for 200.03s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [ 908.001005] Pid: 17553, comm: mdt00_010 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 SMP Tue Jan 25 12:06:24 PST 2022 [ 908.012657] Call Trace: [ 908.015870] [] ptlrpc_set_wait+0x4d8/0x800 [ptlrpc] [ 908.023683] [] ptlrpc_queue_wait+0x83/0x230 [ptlrpc] [ 908.031587] [] osp_remote_sync+0xd3/0x200 [osp] [ 908.039001] [] osp_attr_get+0x48e/0x770 [osp] [ 908.046207] [] osp_object_init+0x16d/0x300 [osp] [ 908.053705] [] lu_object_start.isra.34+0x8b/0x120 [obdclass] [ 908.062393] [] lu_object_find_at+0x234/0xab0 [obdclass] [ 908.070578] [] lu_object_find+0x16/0x20 [obdclass] [ 908.078281] [] mdt_object_find+0x4b/0x170 [mdt] [ 908.085704] [] mdt_getattr_name_lock+0x1078/0x1df0 [mdt] [ 908.093983] [] mdt_intent_getattr+0x2b5/0x480 [mdt] [ 908.101775] [] mdt_intent_opc+0x1ba/0xb50 [mdt] [ 908.109179] [] mdt_intent_policy+0x1a4/0x360 [mdt] [ 908.116879] [] ldlm_lock_enqueue+0x376/0x9b0 [ptlrpc] [ 908.124880] [] ldlm_handle_enqueue0+0xa86/0x1620 [ptlrpc] [ 908.133272] [] tgt_enqueue+0x62/0x210 [ptlrpc] [ 908.140602] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [ 908.148805] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 908.157875] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 908.165392] [] kthread+0xd1/0xe0 [ 908.171334] [] ret_from_fork_nospec_end+0x0/0x39 [ 908.178832] [] 0xffffffffffffffff [ 908.184879] LustreError: dumping log to /tmp/lustre-log.1644342564.17553 [ 909.188719] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 21s! [mdt01_003:10825] [ 909.197845] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 909.277701] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 909.338787] CPU: 13 PID: 10825 Comm: mdt01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 909.353150] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 909.365476] task: ffff98322385d280 ti: ffff983225500000 task.ti: ffff983225500000 [ 909.374311] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 909.385876] RSP: 0018:ffff983225503a50 EFLAGS: 00000246 [ 909.392286] RAX: 0000000000000000 RBX: ffff9832bf340000 RCX: 0000000000690000 [ 909.400733] RDX: ffff9832bf3db8c0 RSI: 0000000000790001 RDI: ffff97f2a83bcf40 [ 909.409180] RBP: ffff983225503a50 R08: ffff9832bf35b8c0 R09: 0000000000000000 [ 909.417629] R10: 0000000000000001 R11: 0000000000000005 R12: ffff983225503a18 [ 909.426076] R13: ffffffffa00ec9b1 R14: ffff9832255039f8 R15: ffff983225503a28 [ 909.434525] FS: 0000000000000000(0000) GS:ffff9832bf340000(0000) knlGS:0000000000000000 [ 909.444039] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 909.450934] CR2: 00002aaaab176a00 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 909.459381] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 909.467829] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 909.476276] Call Trace: [ 909.479488] [] queued_spin_lock_slowpath+0xb/0xf [ 909.486966] [] _raw_spin_lock+0x30/0x40 [ 909.493580] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 909.501262] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 909.509710] [] ? wake_up_atomic_t+0x30/0x30 [ 909.516709] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 909.524117] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 909.532392] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 909.540940] [] ? enqueue_task_fair+0x208/0x6c0 [ 909.548224] [] ? check_preempt_curr+0x80/0xa0 [ 909.555413] [] ? ttwu_do_wakeup+0x19/0x100 [ 909.562340] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 909.570524] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 909.579457] [] ? __getnstimeofday64+0x3f/0xd0 [ 909.586669] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 909.595722] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 909.603784] [] ? __wake_up_common_lock+0x91/0xc0 [ 909.611261] [] ? sched_feat_set+0xf0/0xf0 [ 909.618085] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 909.625587] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 909.634328] [] kthread+0xd1/0xe0 [ 909.640257] [] ? insert_kthread_work+0x40/0x40 [ 909.647541] [] ret_from_fork_nospec_begin+0x21/0x21 [ 909.655310] [] ? insert_kthread_work+0x40/0x40 [ 909.662594] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 914.686369] Lustre: 15547:0:(client.c:2169:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1644342464/real 1644342464] req@ffff9832052dc800 x1724217557268672/t0(0) o5->ls1-OST0020-osc-MDT0000@172.19.3.146@o2ib600:28/4 lens 432/432 e 0 to 1 dl 1644342570 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [ 914.718670] Lustre: 15547:0:(client.c:2169:ptlrpc_expire_one_request()) Skipped 8 previous similar messages [ 914.730039] Lustre: ls1-OST0020-osc-MDT0000: Connection to ls1-OST0020 (at 172.19.3.146@o2ib600) was lost; in progress operations using this service will wait for recovery to complete [ 914.748770] Lustre: Skipped 1 previous similar message [ 914.755012] LustreError: 15547:0:(osp_precreate.c:964:osp_precreate_cleanup_orphans()) ls1-OST0020-osc-MDT0000: cannot cleanup orphans: rc = -11 [ 914.769962] LustreError: 15547:0:(osp_precreate.c:964:osp_precreate_cleanup_orphans()) Skipped 1 previous similar message [ 917.216642] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [mdt00_009:17321] [ 917.225769] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 917.305595] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 917.366652] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 917.381013] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 917.393340] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 917.402176] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 917.413741] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 917.420151] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 917.428597] RDX: ffff9832bf3db8c0 RSI: 0000000000790001 RDI: ffff97f2a83bcf40 [ 917.437044] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 917.445491] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 917.453939] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 917.462386] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 917.471901] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 917.478795] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 917.487242] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 917.495688] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 917.504135] Call Trace: [ 917.507350] [] queued_spin_lock_slowpath+0xb/0xf [ 917.514831] [] _raw_spin_lock+0x30/0x40 [ 917.521447] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 917.529122] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 917.537099] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 917.545647] [] ? wake_up_atomic_t+0x30/0x30 [ 917.552645] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 917.560073] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 917.568335] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 917.576899] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 917.585057] [] ? enqueue_task_fair+0x208/0x6c0 [ 917.592342] [] ? sched_clock_cpu+0xa5/0xe0 [ 917.599238] [] ? check_preempt_curr+0x80/0xa0 [ 917.606424] [] ? ttwu_do_wakeup+0x19/0x100 [ 917.613350] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 917.621529] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 917.630464] [] ? __getnstimeofday64+0x3f/0xd0 [ 917.637677] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 917.646731] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 917.654792] [] ? __wake_up_common_lock+0x91/0xc0 [ 917.662272] [] ? sched_feat_set+0xf0/0xf0 [ 917.669094] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 917.676575] [] ? __switch_to+0xce/0x5a0 [ 917.683203] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 917.691944] [] kthread+0xd1/0xe0 [ 917.697871] [] ? insert_kthread_work+0x40/0x40 [ 917.705158] [] ret_from_fork_nospec_begin+0x21/0x21 [ 917.712927] [] ? insert_kthread_work+0x40/0x40 [ 917.720211] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 920.742359] LustreError: 15513:0:(osp_precreate.c:680:osp_precreate_send()) ls1-OST001e-osc-MDT0000: can't precreate: rc = -11 [ 920.755571] LustreError: 15513:0:(osp_precreate.c:1343:osp_precreate_thread()) ls1-OST001e-osc-MDT0000: cannot precreate objects: rc = -11 [ 921.071086] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [mdt00_013:17594] [ 921.080114] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 921.168097] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [mdt01_022:17601] [ 921.159956] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx [ 921.168098] Modules linked in: [ 921.168099] osp(OE) [ 921.168100] mdd(OE) [ 921.168101] lod(OE) [ 921.168101] mdt(OE) [ 921.168101] lfsck(OE) [ 921.168102] mgs(OE) [ 921.168102] mgc(OE) [ 921.168102] osd_zfs(OE) [ 921.168103] lquota(OE) [ 921.168103] fid(OE) [ 921.168104] fld(OE) [ 921.168104] ptlrpc(OE) [ 921.168104] obdclass(OE) [ 921.168105] ko2iblnd(OE) [ 921.168105] lnet(OE) [ 921.168105] libcfs(OE) [ 921.168105] iTCO_wdt [ 921.168106] iTCO_vendor_support [ 921.168106] sb_edac [ 921.168107] intel_powerclamp [ 921.168107] coretemp [ 921.168108] intel_rapl [ 921.168108] iosf_mbi [ 921.168109] rpcrdma [ 921.168109] ib_iser [ 921.168109] kvm [ 921.168110] irqbypass [ 921.168110] joydev [ 921.168111] pcspkr [ 921.168111] lpc_ich [ 921.168112] i2c_i801 [ 921.168112] ioatdma [ 921.168112] ses [ 921.168113] enclosure [ 921.168113] sg [ 921.168114] ipmi_si [ 921.168114] ipmi_devintf [ 921.168114] ipmi_msghandler [ 921.168115] acpi_power_meter [ 921.168115] dm_round_robin [ 921.168115] acpi_cpufreq [ 921.168116] ib_ipoib [ 921.168116] rdma_ucm [ 921.168117] ib_umad [ 921.168117] sch_fq_codel [ 921.168117] iw_cxgb4 [ 921.168118] rdma_cm [ 921.168118] iw_cm [ 921.168118] ib_cm [ 921.168119] iw_cxgb3 [ 921.168119] zfs(POE) [ 921.168120] binfmt_misc [ 921.168120] zunicode(POE) [ 921.168121] zavl(POE) [ 921.168121] icp(POE) [ 921.168121] zcommon(POE) [ 921.168122] znvpair(POE) [ 921.168122] spl(OE) [ 921.168123] msr_safe(OE) [ 921.168123] ip_tables [ 921.168123] nfsv3 [ 921.168124] nfs_acl [ 921.168124] rpcsec_gss_krb5 [ 921.168125] auth_rpcgss [ 921.168125] nfsv4 [ 921.168125] dns_resolver [ 921.168126] nfs [ 921.168126] lockd [ 921.168126] grace [ 921.168127] fscache [ 921.168127] overlay(T) [ 921.168128] ext4 [ 921.168128] mbcache [ 921.168129] jbd2 [ 921.168129] dm_service_time [ 921.168129] sd_mod [ 921.168130] crc_t10dif [ 921.168130] crct10dif_generic [ 921.168131] mlx5_ib [ 921.168131] ib_uverbs [ 921.168131] ib_core [ 921.168132] be2iscsi [ 921.168132] bnx2i [ 921.168132] cnic [ 921.168133] uio [ 921.168133] cxgb4i [ 921.168134] cxgb4 [ 921.168134] cxgb3i [ 921.168135] cxgb3 [ 921.168135] mdio [ 921.168136] libcxgbi [ 921.168136] libcxgb [ 921.168136] qla4xxx [ 921.168136] iscsi_boot_sysfs [ 921.168137] mgag200 [ 921.168137] i2c_algo_bit [ 921.168138] 8021q [ 921.168138] drm_kms_helper [ 921.168139] garp [ 921.168139] syscopyarea [ 921.168140] mrp [ 921.168140] crct10dif_pclmul [ 921.168140] stp [ 921.168141] sysfillrect [ 921.168141] crct10dif_common [ 921.168141] sysimgblt [ 921.168142] crc32_pclmul [ 921.168142] llc [ 921.168142] fb_sys_fops [ 921.168143] crc32c_intel [ 921.168143] ttm [ 921.168143] ghash_clmulni_intel [ 921.168144] mlx5_core [ 921.168144] drm [ 921.168145] mxm_wmi [ 921.168145] ahci [ 921.168145] ixgbe(OE) [ 921.168146] aesni_intel [ 921.168146] mpt3sas [ 921.168146] mlxfw [ 921.168147] dm_multipath [ 921.168147] lrw [ 921.168147] devlink [ 921.168148] libahci [ 921.168148] gf128mul [ 921.168149] dca [ 921.168149] glue_helper [ 921.168149] ablk_helper [ 921.168150] raid_class [ 921.168150] ptp [ 921.168151] drm_panel_orientation_quirks [ 921.168151] libata [ 921.168152] cryptd [ 921.168152] scsi_transport_sas [ 921.168152] pps_core [ 921.168153] wmi [ 921.168153] sunrpc [ 921.168153] dm_mirror [ 921.168154] dm_region_hash [ 921.168154] dm_log [ 921.168154] dm_mod [ 921.168155] iscsi_tcp [ 921.168155] libiscsi_tcp [ 921.168155] libiscsi [ 921.168156] scsi_transport_iscsi [ 921.168156] fuse [ 921.168156] [ 921.168159] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.168159] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.168160] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 921.168161] RIP: 0010:[] [ 921.168165] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 921.168166] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 921.168166] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 921.168167] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 921.168168] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 921.168168] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 921.168169] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 921.168170] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 921.168171] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.168171] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 921.168172] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.168173] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.168174] Call Trace: [ 921.168178] [] queued_spin_lock_slowpath+0xb/0xf [ 921.168180] [] _raw_spin_lock+0x30/0x40 [ 921.168189] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.168204] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.168206] [] ? wake_up_atomic_t+0x30/0x30 [ 921.168212] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.168246] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.168264] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 921.168283] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 921.168301] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 921.168318] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 921.168346] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 921.168373] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.168375] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.168400] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.168425] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.168427] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.168428] [] ? sched_feat_set+0xf0/0xf0 [ 921.168453] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.168479] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.168481] [] kthread+0xd1/0xe0 [ 921.168482] [] ? insert_kthread_work+0x40/0x40 [ 921.168484] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.168485] [] ? insert_kthread_work+0x40/0x40 [ 921.168486] Code: [ 921.168486] 0d [ 921.168487] 48 [ 921.168487] 98 [ 921.168487] 83 [ 921.168488] e2 [ 921.168488] 30 [ 921.168488] 48 [ 921.168488] 81 [ 921.168489] c2 [ 921.168489] c0 [ 921.168489] b8 [ 921.168489] 01 [ 921.168490] 00 [ 921.168490] 48 [ 921.168490] 03 [ 921.168490] 14 [ 921.168491] c5 [ 921.168491] e0 [ 921.168491] 17 [ 921.168491] d5 [ 921.168492] a0 [ 921.168492] 4c [ 921.168492] 89 [ 921.168493] 02 [ 921.168493] 41 [ 921.168493] 8b [ 921.168493] 40 [ 921.168494] 08 [ 921.168494] 85 [ 921.168494] c0 [ 921.168494] 75 [ 921.168495] 0f [ 921.168495] 0f [ 921.168495] 1f [ 921.168495] 44 [ 921.168496] 00 [ 921.168496] 00 [ 921.168496] f3 [ 921.168496] 90 [ 921.168497] 41 [ 921.168497] 8b [ 921.168497] 40 [ 921.168498] 08 [ 921.168498] <85> [ 921.168498] c0 [ 921.168498] 74 [ 921.168499] f6 [ 921.168499] 4d [ 921.168499] 8b [ 921.168499] 08 [ 921.168500] 4d [ 921.168500] 85 [ 921.168500] c9 [ 921.168500] 74 [ 921.168501] 04 [ 921.168501] 41 [ 921.168501] 0f [ 921.168501] 18 [ 921.168502] 09 [ 921.168502] 8b [ 921.168502] 17 [ 921.168502] 0f [ 921.168503] b7 [ 921.168503] c2 [ 921.168503] [ 921.195099] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [mdt01_016:17554] [ 921.195100] Modules linked in: [ 921.195100] osp(OE) [ 921.195101] mdd(OE) [ 921.195101] lod(OE) [ 921.195102] mdt(OE) [ 921.195102] lfsck(OE) [ 921.195103] mgs(OE) [ 921.195103] mgc(OE) [ 921.195103] osd_zfs(OE) [ 921.195104] lquota(OE) [ 921.195104] fid(OE) [ 921.195104] fld(OE) [ 921.195105] ptlrpc(OE) [ 921.195105] obdclass(OE) [ 921.195106] ko2iblnd(OE) [ 921.195106] lnet(OE) [ 921.195107] libcfs(OE) [ 921.195107] iTCO_wdt [ 921.195107] iTCO_vendor_support [ 921.195108] sb_edac [ 921.195108] intel_powerclamp [ 921.195109] coretemp [ 921.195109] intel_rapl [ 921.195109] iosf_mbi [ 921.195110] rpcrdma [ 921.195110] ib_iser [ 921.195110] kvm [ 921.195111] irqbypass [ 921.195111] joydev [ 921.195112] pcspkr [ 921.195112] lpc_ich [ 921.195112] i2c_i801 [ 921.195113] ioatdma [ 921.195113] ses [ 921.195113] enclosure [ 921.195114] sg [ 921.195114] ipmi_si [ 921.195114] ipmi_devintf [ 921.195115] ipmi_msghandler [ 921.195115] acpi_power_meter [ 921.195116] dm_round_robin [ 921.195116] acpi_cpufreq [ 921.195116] ib_ipoib [ 921.195117] rdma_ucm [ 921.195117] ib_umad [ 921.195117] sch_fq_codel [ 921.195118] iw_cxgb4 [ 921.195118] rdma_cm [ 921.195119] iw_cm [ 921.195119] ib_cm [ 921.195119] iw_cxgb3 [ 921.195120] zfs(POE) [ 921.195120] binfmt_misc [ 921.195121] zunicode(POE) [ 921.195121] zavl(POE) [ 921.195122] icp(POE) [ 921.195122] zcommon(POE) [ 921.195123] znvpair(POE) [ 921.195123] spl(OE) [ 921.195123] msr_safe(OE) [ 921.195124] ip_tables [ 921.195124] nfsv3 [ 921.195125] nfs_acl [ 921.195125] rpcsec_gss_krb5 [ 921.195125] auth_rpcgss [ 921.195126] nfsv4 [ 921.195126] dns_resolver [ 921.195127] nfs [ 921.195127] lockd [ 921.195128] grace [ 921.195128] fscache [ 921.195128] overlay(T) [ 921.195129] ext4 [ 921.195129] mbcache [ 921.195130] jbd2 [ 921.195130] dm_service_time [ 921.195130] sd_mod [ 921.195131] crc_t10dif [ 921.195131] crct10dif_generic [ 921.195132] mlx5_ib [ 921.195132] ib_uverbs [ 921.195132] ib_core [ 921.195133] be2iscsi [ 921.195133] bnx2i [ 921.195134] cnic [ 921.195134] uio [ 921.195134] cxgb4i [ 921.195135] cxgb4 [ 921.195135] cxgb3i [ 921.195135] cxgb3 [ 921.195136] mdio [ 921.195136] libcxgbi [ 921.195136] libcxgb [ 921.195137] qla4xxx [ 921.195137] iscsi_boot_sysfs [ 921.195138] mgag200 [ 921.195138] i2c_algo_bit [ 921.195138] 8021q [ 921.195139] drm_kms_helper [ 921.195139] garp [ 921.195140] syscopyarea [ 921.195140] mrp [ 921.195140] crct10dif_pclmul [ 921.195141] stp [ 921.195141] sysfillrect [ 921.195142] crct10dif_common [ 921.195142] sysimgblt [ 921.195143] crc32_pclmul [ 921.195143] llc [ 921.195143] fb_sys_fops [ 921.195144] crc32c_intel [ 921.195144] ttm [ 921.195145] ghash_clmulni_intel [ 921.195145] mlx5_core [ 921.195145] drm [ 921.195146] mxm_wmi [ 921.195146] ahci [ 921.195146] ixgbe(OE) [ 921.195147] aesni_intel [ 921.195147] mpt3sas [ 921.195148] mlxfw [ 921.195148] dm_multipath [ 921.195149] lrw [ 921.195149] devlink [ 921.195149] libahci [ 921.195150] gf128mul [ 921.195150] dca [ 921.195151] glue_helper [ 921.195151] ablk_helper [ 921.195151] raid_class [ 921.195152] ptp [ 921.195152] drm_panel_orientation_quirks [ 921.195152] libata [ 921.195153] cryptd [ 921.195153] scsi_transport_sas [ 921.195154] pps_core [ 921.195154] wmi [ 921.195154] sunrpc [ 921.195155] dm_mirror [ 921.195155] dm_region_hash [ 921.195155] dm_log [ 921.195156] dm_mod [ 921.195156] iscsi_tcp [ 921.195156] libiscsi_tcp [ 921.195157] libiscsi [ 921.195157] scsi_transport_iscsi [ 921.195158] fuse [ 921.195158] [ 921.195160] CPU: 14 PID: 17554 Comm: mdt01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.195161] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.195162] task: ffff98321c985280 ti: ffff98321b544000 task.ti: ffff98321b544000 [ 921.195163] RIP: 0010:[] [ 921.195165] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 921.195166] RSP: 0018:ffff98321b547ab0 EFLAGS: 00000246 [ 921.195167] RAX: 0000000000000000 RBX: ffff9832bf380000 RCX: 0000000000710000 [ 921.195168] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 921.195169] RBP: ffff98321b547ab0 R08: ffff9832bf39b8c0 R09: 0000000000000000 [ 921.195170] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321b547a78 [ 921.195171] R13: ffffffffa00ec9b1 R14: ffff98321b547a58 R15: ffff98321b547a88 [ 921.195172] FS: 0000000000000000(0000) GS:ffff9832bf380000(0000) knlGS:0000000000000000 [ 921.195173] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.195174] CR2: 00007ffff7fd6000 CR3: 0000003f67e7a000 CR4: 00000000003607e0 [ 921.195175] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.195176] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.195176] Call Trace: [ 921.195178] [] queued_spin_lock_slowpath+0xb/0xf [ 921.195180] [] _raw_spin_lock+0x30/0x40 [ 921.195186] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.195195] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.195197] [] ? wake_up_atomic_t+0x30/0x30 [ 921.195204] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.195229] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.195253] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 921.195255] [] ? check_preempt_curr+0x80/0xa0 [ 921.195278] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 921.195301] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 921.195331] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 921.195360] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.195362] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.195388] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.195414] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.195415] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.195417] [] ? sched_feat_set+0xf0/0xf0 [ 921.195441] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.195466] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.195468] [] kthread+0xd1/0xe0 [ 921.195470] [] ? insert_kthread_work+0x40/0x40 [ 921.195471] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.195473] [] ? insert_kthread_work+0x40/0x40 [ 921.195474] Code: [ 921.195474] 13 [ 921.195474] 48 [ 921.195475] c1 [ 921.195475] ea [ 921.195476] 0d [ 921.195476] 48 [ 921.195476] 98 [ 921.195477] 83 [ 921.195477] e2 [ 921.195477] 30 [ 921.195478] 48 [ 921.195478] 81 [ 921.195478] c2 [ 921.195479] c0 [ 921.195479] b8 [ 921.195480] 01 [ 921.195480] 00 [ 921.195481] 48 [ 921.195481] 03 [ 921.195481] 14 [ 921.195482] c5 [ 921.195482] e0 [ 921.195483] 17 [ 921.195483] d5 [ 921.195483] a0 [ 921.195484] 4c [ 921.195484] 89 [ 921.195485] 02 [ 921.195485] 41 [ 921.195485] 8b [ 921.195486] 40 [ 921.195486] 08 [ 921.195486] 85 [ 921.195487] c0 [ 921.195487] 75 [ 921.195488] 0f [ 921.195488] 0f [ 921.195488] 1f [ 921.195489] 44 [ 921.195489] 00 [ 921.195489] 00 [ 921.195490] f3 [ 921.195490] 90 [ 921.195490] <41> [ 921.195491] 8b [ 921.195491] 40 [ 921.195491] 08 [ 921.195492] 85 [ 921.195492] c0 [ 921.195492] 74 [ 921.195493] f6 [ 921.195493] 4d [ 921.195494] 8b [ 921.195494] 08 [ 921.195494] 4d [ 921.195495] 85 [ 921.195495] c9 [ 921.195496] 74 [ 921.195496] 04 [ 921.195496] 41 [ 921.195497] 0f [ 921.195497] 18 [ 921.195498] 09 [ 921.195498] 8b [ 921.195498] [ 921.199100] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [mdt01_007:17310] [ 921.199100] Modules linked in: [ 921.199101] osp(OE) [ 921.199102] mdd(OE) [ 921.199102] lod(OE) [ 921.199102] mdt(OE) [ 921.199103] lfsck(OE) [ 921.199103] mgs(OE) [ 921.199103] mgc(OE) [ 921.199104] osd_zfs(OE) [ 921.199104] lquota(OE) [ 921.199104] fid(OE) [ 921.199105] fld(OE) [ 921.199105] ptlrpc(OE) [ 921.199105] obdclass(OE) [ 921.199105] ko2iblnd(OE) [ 921.199106] lnet(OE) [ 921.199106] libcfs(OE) [ 921.199106] iTCO_wdt [ 921.199107] iTCO_vendor_support [ 921.199107] sb_edac [ 921.199107] intel_powerclamp [ 921.199108] coretemp [ 921.199108] intel_rapl [ 921.199108] iosf_mbi [ 921.199108] rpcrdma [ 921.199109] ib_iser [ 921.199109] kvm [ 921.199109] irqbypass [ 921.199110] joydev [ 921.199110] pcspkr [ 921.199110] lpc_ich [ 921.199110] i2c_i801 [ 921.199111] ioatdma [ 921.199111] ses [ 921.199111] enclosure [ 921.199112] sg [ 921.199112] ipmi_si [ 921.199112] ipmi_devintf [ 921.199113] ipmi_msghandler [ 921.199113] acpi_power_meter [ 921.199113] dm_round_robin [ 921.199114] acpi_cpufreq [ 921.199114] ib_ipoib [ 921.199114] rdma_ucm [ 921.199115] ib_umad [ 921.199115] sch_fq_codel [ 921.199115] iw_cxgb4 [ 921.199116] rdma_cm [ 921.199116] iw_cm [ 921.199116] ib_cm [ 921.199117] iw_cxgb3 [ 921.199117] zfs(POE) [ 921.199117] binfmt_misc [ 921.199117] zunicode(POE) [ 921.199118] zavl(POE) [ 921.199118] icp(POE) [ 921.199118] zcommon(POE) [ 921.199119] znvpair(POE) [ 921.199119] spl(OE) [ 921.199119] msr_safe(OE) [ 921.199120] ip_tables [ 921.199120] nfsv3 [ 921.199120] nfs_acl [ 921.199120] rpcsec_gss_krb5 [ 921.199121] auth_rpcgss [ 921.199121] nfsv4 [ 921.199121] dns_resolver [ 921.199122] nfs [ 921.199122] lockd [ 921.199122] grace [ 921.199122] fscache [ 921.199123] overlay(T) [ 921.199123] ext4 [ 921.199123] mbcache [ 921.199123] jbd2 [ 921.199124] dm_service_time [ 921.199124] sd_mod [ 921.199124] crc_t10dif [ 921.199125] crct10dif_generic [ 921.199125] mlx5_ib [ 921.199125] ib_uverbs [ 921.199126] ib_core [ 921.199126] be2iscsi [ 921.199126] bnx2i [ 921.199127] cnic [ 921.199127] uio [ 921.199127] cxgb4i [ 921.199127] cxgb4 [ 921.199128] cxgb3i [ 921.199128] cxgb3 [ 921.199128] mdio [ 921.199129] libcxgbi [ 921.199129] libcxgb [ 921.199129] qla4xxx [ 921.199130] iscsi_boot_sysfs [ 921.199130] mgag200 [ 921.199130] i2c_algo_bit [ 921.199131] 8021q [ 921.199131] drm_kms_helper [ 921.199131] garp [ 921.199132] syscopyarea [ 921.199132] mrp [ 921.199132] crct10dif_pclmul [ 921.199133] stp [ 921.199133] sysfillrect [ 921.199133] crct10dif_common [ 921.199133] sysimgblt [ 921.199134] crc32_pclmul [ 921.199134] llc [ 921.199134] fb_sys_fops [ 921.199134] crc32c_intel [ 921.199135] ttm [ 921.199135] ghash_clmulni_intel [ 921.199135] mlx5_core [ 921.199135] drm [ 921.199136] mxm_wmi [ 921.199136] ahci [ 921.199136] ixgbe(OE) [ 921.199137] aesni_intel [ 921.199137] mpt3sas [ 921.199137] mlxfw [ 921.199137] dm_multipath [ 921.199138] lrw [ 921.199138] devlink [ 921.199138] libahci [ 921.199138] gf128mul [ 921.199139] dca [ 921.199139] glue_helper [ 921.199139] ablk_helper [ 921.199140] raid_class [ 921.199140] ptp [ 921.199140] drm_panel_orientation_quirks [ 921.199140] libata [ 921.199141] cryptd [ 921.199141] scsi_transport_sas [ 921.199141] pps_core [ 921.199141] wmi [ 921.199142] sunrpc [ 921.199142] dm_mirror [ 921.199142] dm_region_hash [ 921.199143] dm_log [ 921.199143] dm_mod [ 921.199143] iscsi_tcp [ 921.199144] libiscsi_tcp [ 921.199144] libiscsi [ 921.199144] scsi_transport_iscsi [ 921.199145] fuse [ 921.199145] [ 921.199147] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.199147] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.199148] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 921.199149] RIP: 0010:[] [ 921.199151] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 921.199152] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 921.199152] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 921.199153] RDX: ffff97f2bfa1b8c0 RSI: 0000000000010000 RDI: ffff97f2a83bcf40 [ 921.199153] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 921.199154] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 921.199155] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 921.199155] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 921.199156] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.199157] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 921.199157] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.199158] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.199158] Call Trace: [ 921.199161] [] queued_spin_lock_slowpath+0xb/0xf [ 921.199162] [] _raw_spin_lock+0x30/0x40 [ 921.199167] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.199174] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.199175] [] ? wake_up_atomic_t+0x30/0x30 [ 921.199181] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.199200] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.199218] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 921.199219] [] ? check_preempt_curr+0x80/0xa0 [ 921.199236] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 921.199253] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 921.199276] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 921.199299] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.199300] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.199320] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.199338] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.199339] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.199340] [] ? sched_feat_set+0xf0/0xf0 [ 921.199359] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.199377] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.199378] [] kthread+0xd1/0xe0 [ 921.199380] [] ? insert_kthread_work+0x40/0x40 [ 921.199381] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.199383] [] ? insert_kthread_work+0x40/0x40 [ 921.199383] Code: [ 921.199383] 0d [ 921.199384] 48 [ 921.199384] 98 [ 921.199384] 83 [ 921.199385] e2 [ 921.199385] 30 [ 921.199385] 48 [ 921.199385] 81 [ 921.199385] c2 [ 921.199386] c0 [ 921.199386] b8 [ 921.199386] 01 [ 921.199386] 00 [ 921.199387] 48 [ 921.199387] 03 [ 921.199387] 14 [ 921.199387] c5 [ 921.199388] e0 [ 921.199388] 17 [ 921.199388] d5 [ 921.199388] a0 [ 921.199389] 4c [ 921.199389] 89 [ 921.199389] 02 [ 921.199389] 41 [ 921.199390] 8b [ 921.199390] 40 [ 921.199390] 08 [ 921.199391] 85 [ 921.199391] c0 [ 921.199391] 75 [ 921.199392] 0f [ 921.199392] 0f [ 921.199392] 1f [ 921.199393] 44 [ 921.199393] 00 [ 921.199393] 00 [ 921.199393] f3 [ 921.199394] 90 [ 921.199394] 41 [ 921.199394] 8b [ 921.199395] 40 [ 921.199395] 08 [ 921.199395] <85> [ 921.199396] c0 [ 921.199396] 74 [ 921.199397] f6 [ 921.199397] 4d [ 921.199397] 8b [ 921.199397] 08 [ 921.199398] 4d [ 921.199398] 85 [ 921.199398] c9 [ 921.199399] 74 [ 921.199399] 04 [ 921.199399] 41 [ 921.199399] 0f [ 921.199400] 18 [ 921.199400] 09 [ 921.199400] 8b [ 921.199401] 17 [ 921.199401] 0f [ 921.199401] b7 [ 921.199401] c2 [ 921.199402] [ 921.213102] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ll_mgs_0013:10822] [ 921.213102] Modules linked in: [ 921.213103] osp(OE) [ 921.213103] mdd(OE) [ 921.213104] lod(OE) [ 921.213104] mdt(OE) [ 921.213105] lfsck(OE) [ 921.213105] mgs(OE) [ 921.213105] mgc(OE) [ 921.213106] osd_zfs(OE) [ 921.213106] lquota(OE) [ 921.213106] fid(OE) [ 921.213107] fld(OE) [ 921.213107] ptlrpc(OE) [ 921.213107] obdclass(OE) [ 921.213108] ko2iblnd(OE) [ 921.213108] lnet(OE) [ 921.213108] libcfs(OE) [ 921.213108] iTCO_wdt [ 921.213109] iTCO_vendor_support [ 921.213109] sb_edac [ 921.213109] intel_powerclamp [ 921.213110] coretemp [ 921.213110] intel_rapl [ 921.213110] iosf_mbi [ 921.213110] rpcrdma [ 921.213111] ib_iser [ 921.213111] kvm [ 921.213111] irqbypass [ 921.213112] joydev [ 921.213112] pcspkr [ 921.213112] lpc_ich [ 921.213112] i2c_i801 [ 921.213113] ioatdma [ 921.213113] ses [ 921.213113] enclosure [ 921.213113] sg [ 921.213114] ipmi_si [ 921.213114] ipmi_devintf [ 921.213114] ipmi_msghandler [ 921.213114] acpi_power_meter [ 921.213115] dm_round_robin [ 921.213115] acpi_cpufreq [ 921.213115] ib_ipoib [ 921.213116] rdma_ucm [ 921.213116] ib_umad [ 921.213116] sch_fq_codel [ 921.213116] iw_cxgb4 [ 921.213117] rdma_cm [ 921.213117] iw_cm [ 921.213117] ib_cm [ 921.213117] iw_cxgb3 [ 921.213118] zfs(POE) [ 921.213118] binfmt_misc [ 921.213118] zunicode(POE) [ 921.213119] zavl(POE) [ 921.213119] icp(POE) [ 921.213119] zcommon(POE) [ 921.213120] znvpair(POE) [ 921.213120] spl(OE) [ 921.213120] msr_safe(OE) [ 921.213120] ip_tables [ 921.213121] nfsv3 [ 921.213121] nfs_acl [ 921.213121] rpcsec_gss_krb5 [ 921.213122] auth_rpcgss [ 921.213122] nfsv4 [ 921.213122] dns_resolver [ 921.213122] nfs [ 921.213123] lockd [ 921.213123] grace [ 921.213123] fscache [ 921.213124] overlay(T) [ 921.213124] ext4 [ 921.213124] mbcache [ 921.213125] jbd2 [ 921.213125] dm_service_time [ 921.213125] sd_mod [ 921.213126] crc_t10dif [ 921.213126] crct10dif_generic [ 921.213126] mlx5_ib [ 921.213127] ib_uverbs [ 921.213127] ib_core [ 921.213127] be2iscsi [ 921.213128] bnx2i [ 921.213128] cnic [ 921.213128] uio [ 921.213128] cxgb4i [ 921.213128] cxgb4 [ 921.213129] cxgb3i [ 921.213129] cxgb3 [ 921.213129] mdio [ 921.213130] libcxgbi [ 921.213130] libcxgb [ 921.213130] qla4xxx [ 921.213131] iscsi_boot_sysfs [ 921.213131] mgag200 [ 921.213132] i2c_algo_bit [ 921.213132] 8021q [ 921.213132] drm_kms_helper [ 921.213133] garp [ 921.213133] syscopyarea [ 921.213134] mrp [ 921.213134] crct10dif_pclmul [ 921.213134] stp [ 921.213135] sysfillrect [ 921.213135] crct10dif_common [ 921.213135] sysimgblt [ 921.213136] crc32_pclmul [ 921.213136] llc [ 921.213136] fb_sys_fops [ 921.213137] crc32c_intel [ 921.213137] ttm [ 921.213138] ghash_clmulni_intel [ 921.213138] mlx5_core [ 921.213139] drm [ 921.213139] mxm_wmi [ 921.213139] ahci [ 921.213140] ixgbe(OE) [ 921.213140] aesni_intel [ 921.213141] mpt3sas [ 921.213141] mlxfw [ 921.213142] dm_multipath [ 921.213142] lrw [ 921.213142] devlink [ 921.213143] libahci [ 921.213143] gf128mul [ 921.213144] dca [ 921.213144] glue_helper [ 921.213145] ablk_helper [ 921.213145] raid_class [ 921.213146] ptp [ 921.213146] drm_panel_orientation_quirks [ 921.213146] libata [ 921.213147] cryptd [ 921.213147] scsi_transport_sas [ 921.213148] pps_core [ 921.213148] wmi [ 921.213149] sunrpc [ 921.213149] dm_mirror [ 921.213150] dm_region_hash [ 921.213150] dm_log [ 921.213151] dm_mod [ 921.213151] iscsi_tcp [ 921.213151] libiscsi_tcp [ 921.213152] libiscsi [ 921.213153] scsi_transport_iscsi [ 921.213153] fuse [ 921.213153] [ 921.213155] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.213156] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.213157] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 921.213157] RIP: 0010:[] [ 921.213161] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 921.213162] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 921.213162] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 921.213163] RDX: ffff9832bf59b8c0 RSI: 0000000000f10000 RDI: ffff97f2a83bcf40 [ 921.213163] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 921.213164] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 921.213165] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 921.213165] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 921.213166] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.213167] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 921.213168] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.213168] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.213168] Call Trace: [ 921.213172] [] queued_spin_lock_slowpath+0xb/0xf [ 921.213175] [] _raw_spin_lock+0x30/0x40 [ 921.213187] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.213200] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.213202] [] ? wake_up_atomic_t+0x30/0x30 [ 921.213207] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.213244] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.213261] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 921.213262] [] ? enqueue_task_fair+0x208/0x6c0 [ 921.213264] [] ? check_preempt_curr+0x80/0xa0 [ 921.213265] [] ? ttwu_do_wakeup+0x19/0x100 [ 921.213293] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 921.213315] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.213317] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.213336] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.213355] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.213356] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.213357] [] ? sched_feat_set+0xf0/0xf0 [ 921.213376] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.213378] [] ? __switch_to+0xce/0x5a0 [ 921.213396] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.213398] [] kthread+0xd1/0xe0 [ 921.213399] [] ? insert_kthread_work+0x40/0x40 [ 921.213401] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.213402] [] ? insert_kthread_work+0x40/0x40 [ 921.213403] Code: [ 921.213403] 0d [ 921.213404] 48 [ 921.213404] 98 [ 921.213404] 83 [ 921.213404] e2 [ 921.213405] 30 [ 921.213405] 48 [ 921.213405] 81 [ 921.213405] c2 [ 921.213406] c0 [ 921.213406] b8 [ 921.213406] 01 [ 921.213407] 00 [ 921.213407] 48 [ 921.213407] 03 [ 921.213407] 14 [ 921.213408] c5 [ 921.213408] e0 [ 921.213408] 17 [ 921.213409] d5 [ 921.213409] a0 [ 921.213409] 4c [ 921.213409] 89 [ 921.213410] 02 [ 921.213410] 41 [ 921.213410] 8b [ 921.213410] 40 [ 921.213411] 08 [ 921.213411] 85 [ 921.213411] c0 [ 921.213411] 75 [ 921.213412] 0f [ 921.213412] 0f [ 921.213412] 1f [ 921.213413] 44 [ 921.213413] 00 [ 921.213413] 00 [ 921.213413] f3 [ 921.213414] 90 [ 921.213414] 41 [ 921.213414] 8b [ 921.213414] 40 [ 921.213415] 08 [ 921.213415] <85> [ 921.213416] c0 [ 921.213416] 74 [ 921.213416] f6 [ 921.213416] 4d [ 921.213417] 8b [ 921.213417] 08 [ 921.213417] 4d [ 921.213417] 85 [ 921.213418] c9 [ 921.213418] 74 [ 921.213418] 04 [ 921.213418] 41 [ 921.213419] 0f [ 921.213419] 18 [ 921.213419] 09 [ 921.213419] 8b [ 921.213420] 17 [ 921.213420] 0f [ 921.213420] b7 [ 921.213420] c2 [ 921.213421] [ 921.219103] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ll_mgs_0014:10826] [ 921.219103] Modules linked in: [ 921.219104] osp(OE) [ 921.219104] mdd(OE) [ 921.219105] lod(OE) [ 921.219105] mdt(OE) [ 921.219105] lfsck(OE) [ 921.219106] mgs(OE) [ 921.219106] mgc(OE) [ 921.219106] osd_zfs(OE) [ 921.219106] lquota(OE) [ 921.219107] fid(OE) [ 921.219107] fld(OE) [ 921.219107] ptlrpc(OE) [ 921.219108] obdclass(OE) [ 921.219108] ko2iblnd(OE) [ 921.219108] lnet(OE) [ 921.219109] libcfs(OE) [ 921.219109] iTCO_wdt [ 921.219109] iTCO_vendor_support [ 921.219110] sb_edac [ 921.219110] intel_powerclamp [ 921.219110] coretemp [ 921.219110] intel_rapl [ 921.219111] iosf_mbi [ 921.219111] rpcrdma [ 921.219111] ib_iser [ 921.219111] kvm [ 921.219112] irqbypass [ 921.219112] joydev [ 921.219112] pcspkr [ 921.219113] lpc_ich [ 921.219113] i2c_i801 [ 921.219113] ioatdma [ 921.219113] ses [ 921.219114] enclosure [ 921.219114] sg [ 921.219114] ipmi_si [ 921.219114] ipmi_devintf [ 921.219115] ipmi_msghandler [ 921.219115] acpi_power_meter [ 921.219115] dm_round_robin [ 921.219116] acpi_cpufreq [ 921.219116] ib_ipoib [ 921.219116] rdma_ucm [ 921.219117] ib_umad [ 921.219117] sch_fq_codel [ 921.219117] iw_cxgb4 [ 921.219117] rdma_cm [ 921.219118] iw_cm [ 921.219118] ib_cm [ 921.219118] iw_cxgb3 [ 921.219119] zfs(POE) [ 921.219119] binfmt_misc [ 921.219119] zunicode(POE) [ 921.219120] zavl(POE) [ 921.219120] icp(POE) [ 921.219120] zcommon(POE) [ 921.219121] znvpair(POE) [ 921.219121] spl(OE) [ 921.219121] msr_safe(OE) [ 921.219122] ip_tables [ 921.219122] nfsv3 [ 921.219122] nfs_acl [ 921.219122] rpcsec_gss_krb5 [ 921.219123] auth_rpcgss [ 921.219123] nfsv4 [ 921.219123] dns_resolver [ 921.219124] nfs [ 921.219124] lockd [ 921.219124] grace [ 921.219124] fscache [ 921.219125] overlay(T) [ 921.219125] ext4 [ 921.219125] mbcache [ 921.219126] jbd2 [ 921.219126] dm_service_time [ 921.219126] sd_mod [ 921.219126] crc_t10dif [ 921.219127] crct10dif_generic [ 921.219127] mlx5_ib [ 921.219127] ib_uverbs [ 921.219128] ib_core [ 921.219128] be2iscsi [ 921.219128] bnx2i [ 921.219128] cnic [ 921.219129] uio [ 921.219129] cxgb4i [ 921.219129] cxgb4 [ 921.219130] cxgb3i [ 921.219130] cxgb3 [ 921.219130] mdio [ 921.219130] libcxgbi [ 921.219131] libcxgb [ 921.219131] qla4xxx [ 921.219131] iscsi_boot_sysfs [ 921.219131] mgag200 [ 921.219132] i2c_algo_bit [ 921.219132] 8021q [ 921.219132] drm_kms_helper [ 921.219133] garp [ 921.219133] syscopyarea [ 921.219133] mrp [ 921.219133] crct10dif_pclmul [ 921.219134] stp [ 921.219134] sysfillrect [ 921.219134] crct10dif_common [ 921.219134] sysimgblt [ 921.219135] crc32_pclmul [ 921.219135] llc [ 921.219135] fb_sys_fops [ 921.219135] crc32c_intel [ 921.219136] ttm [ 921.219136] ghash_clmulni_intel [ 921.219136] mlx5_core [ 921.219137] drm [ 921.219137] mxm_wmi [ 921.219137] ahci [ 921.219137] ixgbe(OE) [ 921.219138] aesni_intel [ 921.219138] mpt3sas [ 921.219138] mlxfw [ 921.219139] dm_multipath [ 921.219139] lrw [ 921.219139] devlink [ 921.219140] libahci [ 921.219140] gf128mul [ 921.219140] dca [ 921.219140] glue_helper [ 921.219141] ablk_helper [ 921.219141] raid_class [ 921.219141] ptp [ 921.219142] drm_panel_orientation_quirks [ 921.219142] libata [ 921.219142] cryptd [ 921.219142] scsi_transport_sas [ 921.219143] pps_core [ 921.219143] wmi [ 921.219143] sunrpc [ 921.219144] dm_mirror [ 921.219144] dm_region_hash [ 921.219145] dm_log [ 921.219145] dm_mod [ 921.219145] iscsi_tcp [ 921.219145] libiscsi_tcp [ 921.219146] libiscsi [ 921.219146] scsi_transport_iscsi [ 921.219146] fuse [ 921.219146] [ 921.219148] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.219149] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.219149] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 921.219150] RIP: 0010:[] [ 921.219152] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 921.219153] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 921.219154] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 921.219154] RDX: ffff97f2bfadb8c0 RSI: 0000000000190000 RDI: ffff97f2a83bcf40 [ 921.219155] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 921.219156] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 921.219156] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 921.219157] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 921.219158] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.219159] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 921.219159] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.219160] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.219160] Call Trace: [ 921.219163] [] queued_spin_lock_slowpath+0xb/0xf [ 921.219164] [] _raw_spin_lock+0x30/0x40 [ 921.219169] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.219176] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.219178] [] ? wake_up_atomic_t+0x30/0x30 [ 921.219183] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.219207] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.219231] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 921.219233] [] ? check_preempt_curr+0x80/0xa0 [ 921.219257] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 921.219280] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 921.219308] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 921.219334] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.219336] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.219361] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.219386] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.219388] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.219389] [] ? sched_feat_set+0xf0/0xf0 [ 921.219414] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.219415] [] ? __switch_to+0xce/0x5a0 [ 921.219441] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.219443] [] kthread+0xd1/0xe0 [ 921.219444] [] ? insert_kthread_work+0x40/0x40 [ 921.219446] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.219447] [] ? insert_kthread_work+0x40/0x40 [ 921.219447] Code: [ 921.219448] 13 [ 921.219448] 48 [ 921.219449] c1 [ 921.219449] ea [ 921.219449] 0d [ 921.219449] 48 [ 921.219450] 98 [ 921.219450] 83 [ 921.219450] e2 [ 921.219450] 30 [ 921.219451] 48 [ 921.219451] 81 [ 921.219451] c2 [ 921.219451] c0 [ 921.219452] b8 [ 921.219452] 01 [ 921.219452] 00 [ 921.219452] 48 [ 921.219453] 03 [ 921.219453] 14 [ 921.219453] c5 [ 921.219453] e0 [ 921.219454] 17 [ 921.219454] d5 [ 921.219454] a0 [ 921.219454] 4c [ 921.219455] 89 [ 921.219455] 02 [ 921.219455] 41 [ 921.219455] 8b [ 921.219456] 40 [ 921.219456] 08 [ 921.219456] 85 [ 921.219456] c0 [ 921.219456] 75 [ 921.219457] 0f [ 921.219457] 0f [ 921.219457] 1f [ 921.219457] 44 [ 921.219458] 00 [ 921.219458] 00 [ 921.219458] f3 [ 921.219458] 90 [ 921.219459] <41> [ 921.219459] 8b [ 921.219459] 40 [ 921.219460] 08 [ 921.219460] 85 [ 921.219460] c0 [ 921.219460] 74 [ 921.219460] f6 [ 921.219461] 4d [ 921.219461] 8b [ 921.219461] 08 [ 921.219462] 4d [ 921.219462] 85 [ 921.219462] c9 [ 921.219462] 74 [ 921.219463] 04 [ 921.219463] 41 [ 921.219463] 0f [ 921.219463] 18 [ 921.219464] 09 [ 921.219464] 8b [ 921.219464] [ 921.231104] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 22s! [mdt01_014:17318] [ 921.231104] Modules linked in: [ 921.231105] osp(OE) [ 921.231106] mdd(OE) [ 921.231106] lod(OE) [ 921.231106] mdt(OE) [ 921.231107] lfsck(OE) [ 921.231107] mgs(OE) [ 921.231107] mgc(OE) [ 921.231108] osd_zfs(OE) [ 921.231108] lquota(OE) [ 921.231108] fid(OE) [ 921.231109] fld(OE) [ 921.231109] ptlrpc(OE) [ 921.231109] obdclass(OE) [ 921.231110] ko2iblnd(OE) [ 921.231110] lnet(OE) [ 921.231110] libcfs(OE) [ 921.231111] iTCO_wdt [ 921.231111] iTCO_vendor_support [ 921.231111] sb_edac [ 921.231112] intel_powerclamp [ 921.231112] coretemp [ 921.231112] intel_rapl [ 921.231112] iosf_mbi [ 921.231113] rpcrdma [ 921.231113] ib_iser [ 921.231113] kvm [ 921.231114] irqbypass [ 921.231114] joydev [ 921.231114] pcspkr [ 921.231114] lpc_ich [ 921.231115] i2c_i801 [ 921.231115] ioatdma [ 921.231115] ses [ 921.231116] enclosure [ 921.231116] sg [ 921.231116] ipmi_si [ 921.231117] ipmi_devintf [ 921.231117] ipmi_msghandler [ 921.231117] acpi_power_meter [ 921.231118] dm_round_robin [ 921.231118] acpi_cpufreq [ 921.231118] ib_ipoib [ 921.231118] rdma_ucm [ 921.231119] ib_umad [ 921.231119] sch_fq_codel [ 921.231119] iw_cxgb4 [ 921.231120] rdma_cm [ 921.231120] iw_cm [ 921.231120] ib_cm [ 921.231120] iw_cxgb3 [ 921.231121] zfs(POE) [ 921.231121] binfmt_misc [ 921.231121] zunicode(POE) [ 921.231122] zavl(POE) [ 921.231122] icp(POE) [ 921.231122] zcommon(POE) [ 921.231123] znvpair(POE) [ 921.231123] spl(OE) [ 921.231123] msr_safe(OE) [ 921.231124] ip_tables [ 921.231124] nfsv3 [ 921.231124] nfs_acl [ 921.231124] rpcsec_gss_krb5 [ 921.231125] auth_rpcgss [ 921.231125] nfsv4 [ 921.231125] dns_resolver [ 921.231126] nfs [ 921.231126] lockd [ 921.231126] grace [ 921.231126] fscache [ 921.231127] overlay(T) [ 921.231127] ext4 [ 921.231127] mbcache [ 921.231127] jbd2 [ 921.231128] dm_service_time [ 921.231128] sd_mod [ 921.231128] crc_t10dif [ 921.231129] crct10dif_generic [ 921.231129] mlx5_ib [ 921.231129] ib_uverbs [ 921.231129] ib_core [ 921.231130] be2iscsi [ 921.231130] bnx2i [ 921.231130] cnic [ 921.231130] uio [ 921.231131] cxgb4i [ 921.231131] cxgb4 [ 921.231132] cxgb3i [ 921.231132] cxgb3 [ 921.231132] mdio [ 921.231132] libcxgbi [ 921.231133] libcxgb [ 921.231133] qla4xxx [ 921.231133] iscsi_boot_sysfs [ 921.231134] mgag200 [ 921.231134] i2c_algo_bit [ 921.231135] 8021q [ 921.231135] drm_kms_helper [ 921.231135] garp [ 921.231136] syscopyarea [ 921.231136] mrp [ 921.231136] crct10dif_pclmul [ 921.231136] stp [ 921.231137] sysfillrect [ 921.231137] crct10dif_common [ 921.231137] sysimgblt [ 921.231138] crc32_pclmul [ 921.231138] llc [ 921.231138] fb_sys_fops [ 921.231138] crc32c_intel [ 921.231139] ttm [ 921.231139] ghash_clmulni_intel [ 921.231140] mlx5_core [ 921.231140] drm [ 921.231140] mxm_wmi [ 921.231141] ahci [ 921.231141] ixgbe(OE) [ 921.231141] aesni_intel [ 921.231142] mpt3sas [ 921.231142] mlxfw [ 921.231142] dm_multipath [ 921.231142] lrw [ 921.231143] devlink [ 921.231143] libahci [ 921.231143] gf128mul [ 921.231143] dca [ 921.231144] glue_helper [ 921.231144] ablk_helper [ 921.231144] raid_class [ 921.231144] ptp [ 921.231145] drm_panel_orientation_quirks [ 921.231145] libata [ 921.231145] cryptd [ 921.231146] scsi_transport_sas [ 921.231146] pps_core [ 921.231146] wmi [ 921.231146] sunrpc [ 921.231147] dm_mirror [ 921.231147] dm_region_hash [ 921.231147] dm_log [ 921.231147] dm_mod [ 921.231148] iscsi_tcp [ 921.231148] libiscsi_tcp [ 921.231148] libiscsi [ 921.231149] scsi_transport_iscsi [ 921.231149] fuse [ 921.231149] [ 921.231150] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.231151] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.231152] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 921.231152] RIP: 0010:[] [ 921.231154] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 921.231155] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 921.231156] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 921.231157] RDX: ffff9832bf35b8c0 RSI: 0000000000690001 RDI: ffff97f2a83bcf40 [ 921.231157] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 921.231158] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 921.231158] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 921.231159] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 921.231160] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.231161] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 921.231161] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.231162] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.231162] Call Trace: [ 921.231164] [] queued_spin_lock_slowpath+0xb/0xf [ 921.231166] [] _raw_spin_lock+0x30/0x40 [ 921.231172] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.231179] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.231180] [] ? wake_up_atomic_t+0x30/0x30 [ 921.231187] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.231211] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.231235] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 921.231260] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 921.231284] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 921.231307] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 921.231335] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 921.231361] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.231363] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.231388] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.231413] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.231415] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.231416] [] ? sched_feat_set+0xf0/0xf0 [ 921.231441] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.231467] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.231468] [] kthread+0xd1/0xe0 [ 921.231470] [] ? insert_kthread_work+0x40/0x40 [ 921.231471] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.231473] [] ? insert_kthread_work+0x40/0x40 [ 921.231473] Code: [ 921.231473] c1 [ 921.231474] e8 [ 921.231474] 13 [ 921.231474] 48 [ 921.231474] c1 [ 921.231475] ea [ 921.231475] 0d [ 921.231475] 48 [ 921.231475] 98 [ 921.231476] 83 [ 921.231476] e2 [ 921.231476] 30 [ 921.231477] 48 [ 921.231477] 81 [ 921.231477] c2 [ 921.231477] c0 [ 921.231478] b8 [ 921.231478] 01 [ 921.231478] 00 [ 921.231478] 48 [ 921.231479] 03 [ 921.231479] 14 [ 921.231479] c5 [ 921.231479] e0 [ 921.231480] 17 [ 921.231480] d5 [ 921.231480] a0 [ 921.231480] 4c [ 921.231481] 89 [ 921.231481] 02 [ 921.231481] 41 [ 921.231482] 8b [ 921.231482] 40 [ 921.231482] 08 [ 921.231482] 85 [ 921.231483] c0 [ 921.231483] 75 [ 921.231483] 0f [ 921.231484] 0f [ 921.231484] 1f [ 921.231484] 44 [ 921.231484] 00 [ 921.231485] 00 [ 921.231485] [ 921.231485] 90 [ 921.231485] 41 [ 921.231486] 8b [ 921.231486] 40 [ 921.231486] 08 [ 921.231486] 85 [ 921.231487] c0 [ 921.231487] 74 [ 921.231487] f6 [ 921.231487] 4d [ 921.231488] 8b [ 921.231488] 08 [ 921.231488] 4d [ 921.231488] 85 [ 921.231489] c9 [ 921.231489] 74 [ 921.231489] 04 [ 921.231489] 41 [ 921.231490] 0f [ 921.231490] 18 [ 921.231490] [ 921.236104] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [mdt01_017:17555] [ 921.236104] Modules linked in: [ 921.236105] osp(OE) [ 921.236105] mdd(OE) [ 921.236106] lod(OE) [ 921.236106] mdt(OE) [ 921.236106] lfsck(OE) [ 921.236107] mgs(OE) [ 921.236107] mgc(OE) [ 921.236108] osd_zfs(OE) [ 921.236108] lquota(OE) [ 921.236109] fid(OE) [ 921.236109] fld(OE) [ 921.236109] ptlrpc(OE) [ 921.236110] obdclass(OE) [ 921.236110] ko2iblnd(OE) [ 921.236111] lnet(OE) [ 921.236111] libcfs(OE) [ 921.236111] iTCO_wdt [ 921.236112] iTCO_vendor_support [ 921.236112] sb_edac [ 921.236113] intel_powerclamp [ 921.236113] coretemp [ 921.236114] intel_rapl [ 921.236114] iosf_mbi [ 921.236114] rpcrdma [ 921.236115] ib_iser [ 921.236115] kvm [ 921.236115] irqbypass [ 921.236116] joydev [ 921.236116] pcspkr [ 921.236117] lpc_ich [ 921.236117] i2c_i801 [ 921.236117] ioatdma [ 921.236118] ses [ 921.236118] enclosure [ 921.236119] sg [ 921.236119] ipmi_si [ 921.236119] ipmi_devintf [ 921.236120] ipmi_msghandler [ 921.236120] acpi_power_meter [ 921.236121] dm_round_robin [ 921.236121] acpi_cpufreq [ 921.236121] ib_ipoib [ 921.236122] rdma_ucm [ 921.236122] ib_umad [ 921.236123] sch_fq_codel [ 921.236123] iw_cxgb4 [ 921.236123] rdma_cm [ 921.236124] iw_cm [ 921.236124] ib_cm [ 921.236125] iw_cxgb3 [ 921.236125] zfs(POE) [ 921.236126] binfmt_misc [ 921.236126] zunicode(POE) [ 921.236127] zavl(POE) [ 921.236127] icp(POE) [ 921.236128] zcommon(POE) [ 921.236128] znvpair(POE) [ 921.236128] spl(OE) [ 921.236129] msr_safe(OE) [ 921.236129] ip_tables [ 921.236130] nfsv3 [ 921.236130] nfs_acl [ 921.236131] rpcsec_gss_krb5 [ 921.236131] auth_rpcgss [ 921.236131] nfsv4 [ 921.236132] dns_resolver [ 921.236132] nfs [ 921.236133] lockd [ 921.236133] grace [ 921.236133] fscache [ 921.236134] overlay(T) [ 921.236135] ext4 [ 921.236135] mbcache [ 921.236135] jbd2 [ 921.236136] dm_service_time [ 921.236136] sd_mod [ 921.236136] crc_t10dif [ 921.236137] crct10dif_generic [ 921.236137] mlx5_ib [ 921.236138] ib_uverbs [ 921.236138] ib_core [ 921.236139] be2iscsi [ 921.236139] bnx2i [ 921.236139] cnic [ 921.236140] uio [ 921.236140] cxgb4i [ 921.236140] cxgb4 [ 921.236141] cxgb3i [ 921.236141] cxgb3 [ 921.236142] mdio [ 921.236142] libcxgbi [ 921.236142] libcxgb [ 921.236143] qla4xxx [ 921.236143] iscsi_boot_sysfs [ 921.236144] mgag200 [ 921.236144] i2c_algo_bit [ 921.236145] 8021q [ 921.236145] drm_kms_helper [ 921.236145] garp [ 921.236146] syscopyarea [ 921.236146] mrp [ 921.236147] crct10dif_pclmul [ 921.236147] stp [ 921.236147] sysfillrect [ 921.236148] crct10dif_common [ 921.236148] sysimgblt [ 921.236149] crc32_pclmul [ 921.236149] llc [ 921.236149] fb_sys_fops [ 921.236150] crc32c_intel [ 921.236150] ttm [ 921.236150] ghash_clmulni_intel [ 921.236151] mlx5_core [ 921.236151] drm [ 921.236152] mxm_wmi [ 921.236152] ahci [ 921.236153] ixgbe(OE) [ 921.236153] aesni_intel [ 921.236153] mpt3sas [ 921.236154] mlxfw [ 921.236154] dm_multipath [ 921.236154] lrw [ 921.236155] devlink [ 921.236155] libahci [ 921.236156] gf128mul [ 921.236156] dca [ 921.236157] glue_helper [ 921.236157] ablk_helper [ 921.236157] raid_class [ 921.236158] ptp [ 921.236158] drm_panel_orientation_quirks [ 921.236159] libata [ 921.236159] cryptd [ 921.236160] scsi_transport_sas [ 921.236160] pps_core [ 921.236160] wmi [ 921.236161] sunrpc [ 921.236161] dm_mirror [ 921.236161] dm_region_hash [ 921.236162] dm_log [ 921.236162] dm_mod [ 921.236163] iscsi_tcp [ 921.236163] libiscsi_tcp [ 921.236163] libiscsi [ 921.236164] scsi_transport_iscsi [ 921.236164] fuse [ 921.236164] [ 921.236166] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.236167] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.236168] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 921.236169] RIP: 0010:[] [ 921.236171] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 921.236172] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000246 [ 921.236173] RAX: 0000000000000000 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 921.236174] RDX: ffff97f2bfd1b8c0 RSI: 0000000000a10000 RDI: ffff97f2a83bcf40 [ 921.236174] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: 0000000000000000 [ 921.236175] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 921.236176] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 921.236178] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 921.236179] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.236180] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 921.236180] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.236181] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.236182] Call Trace: [ 921.236184] [] queued_spin_lock_slowpath+0xb/0xf [ 921.236185] [] _raw_spin_lock+0x30/0x40 [ 921.236192] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.236200] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.236202] [] ? wake_up_atomic_t+0x30/0x30 [ 921.236210] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.236234] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.236258] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 921.236260] [] ? check_preempt_curr+0x80/0xa0 [ 921.236283] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 921.236306] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 921.236337] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 921.236365] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.236367] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.236392] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.236417] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.236419] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.236480] iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 921.236482] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.236483] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.236483] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 921.236486] RIP: 0010:[] [] _raw_spin_lock+0x20/0x40 [ 921.236487] RSP: 0018:ffff98321b877a78 EFLAGS: 00000287 [ 921.236487] RAX: 0000000000290000 RBX: ffff97f2bfa9b8c0 RCX: ffff98321b877fd8 [ 921.236488] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff97f2a83bcf40 [ 921.236488] RBP: ffff98321b877a98 R08: ffff98321b877ae8 R09: ffff9832bf4db8c0 [ 921.236489] R10: 0000000000000001 R11: 000000000000242e R12: 0000000000000001 [ 921.236490] R13: 000000000000242e R14: ffff97f2bfa80000 R15: ffff98321b877a50 [ 921.236490] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 921.236491] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.236492] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 921.236492] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.236493] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.236493] Call Trace: [ 921.236499] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 921.236503] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 921.236509] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 921.236511] [] ? wake_up_atomic_t+0x30/0x30 [ 921.236517] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.236536] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.236552] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 921.236570] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 921.236571] [] ? enqueue_task_fair+0x208/0x6c0 [ 921.236573] [] ? sched_clock_cpu+0xa5/0xe0 [ 921.236574] [] ? check_preempt_curr+0x80/0xa0 [ 921.236575] [] ? ttwu_do_wakeup+0x19/0x100 [ 921.236598] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 921.236620] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.236621] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.236641] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.236661] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.236663] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.236663] [] ? sched_feat_set+0xf0/0xf0 [ 921.236682] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.236683] [] ? __switch_to+0xce/0x5a0 [ 921.236702] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.236703] [] kthread+0xd1/0xe0 [ 921.236705] [] ? insert_kthread_work+0x40/0x40 [ 921.236706] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.236707] [] ? insert_kthread_work+0x40/0x40 [ 921.236721] Code: 5d c3 66 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 65 48 8b 04 25 b8 0e 01 00 83 80 44 c0 ff ff 01 31 c0 ba 01 00 00 00 f0 0f b1 17 <85> c0 75 01 c3 55 89 c6 48 89 e5 e8 db 0f ff ff 5d c3 0f 1f 40 [ 921.238104] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 22s! [mdt01_018:17592] [ 921.238132] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 921.238153] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 921.238155] CPU: 30 PID: 17592 Comm: mdt01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 921.238156] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 921.238157] task: ffff983230579080 ti: ffff9832469b0000 task.ti: ffff9832469b0000 [ 921.238160] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 921.238160] RSP: 0018:ffff9832469b3a50 EFLAGS: 00000246 [ 921.238161] RAX: 0000000000000000 RBX: ffff9832bf580000 RCX: 0000000000f10000 [ 921.238162] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 921.238163] RBP: ffff9832469b3a50 R08: ffff9832bf59b8c0 R09: 0000000000000000 [ 921.238164] R10: 0000000000000002 R11: ffff983211336600 R12: ffff9832469b3a18 [ 921.238165] R13: ffffffffa00ec9b1 R14: ffff9832469b39f8 R15: ffff9832469b3a28 [ 921.238166] FS: 0000000000000000(0000) GS:ffff9832bf580000(0000) knlGS:0000000000000000 [ 921.238167] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 921.238167] CR2: 00007ffff7fc5010 CR3: 0000003f74514000 CR4: 00000000003607e0 [ 921.238168] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 921.238169] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 921.238169] Call Trace: [ 921.238171] [] queued_spin_lock_slowpath+0xb/0xf [ 921.238173] [] _raw_spin_lock+0x30/0x40 [ 921.238179] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 921.238187] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 921.238189] [] ? wake_up_atomic_t+0x30/0x30 [ 921.238197] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 921.238221] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 921.238244] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 921.238246] [] ? check_preempt_curr+0x80/0xa0 [ 921.238247] [] ? ttwu_do_wakeup+0x19/0x100 [ 921.238278] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 921.238306] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 921.238308] [] ? __getnstimeofday64+0x3f/0xd0 [ 921.238334] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 921.238358] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 921.238360] [] ? __wake_up_common_lock+0x91/0xc0 [ 921.238361] [] ? sched_feat_set+0xf0/0xf0 [ 921.238386] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 921.238410] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 921.238412] [] kthread+0xd1/0xe0 [ 921.238414] [] ? insert_kthread_work+0x40/0x40 [ 921.238415] [] ret_from_fork_nospec_begin+0x21/0x21 [ 921.238417] [] ? insert_kthread_work+0x40/0x40 [ 921.238436] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 925.486281] [] ? sched_feat_set+0xf0/0xf0 [ 925.493109] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 925.502131] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 925.512400] [] kthread+0xd1/0xe0 [ 925.518330] [] ? insert_kthread_work+0x40/0x40 [ 925.527162] [] ret_from_fork_nospec_begin+0x21/0x21 [ 925.534935] [] ? insert_kthread_work+0x40/0x40 [ 925.542220] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 925.564616] Lustre: ls1-MDT0001-osp-MDT0000: Connection to ls1-MDT0001 (at 172.19.3.99@o2ib600) was lost; in progress operations using this service will wait for recovery to complete [ 925.584738] Lustre: Skipped 5 previous similar messages [ 928.463949] Pid: 10826, comm: ll_mgs_0014 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 SMP Tue Jan 25 12:06:24 PST 2022 [ 928.475794] Call Trace: [ 928.479010] [] 0xffffffffffffffff [ 928.485062] LustreError: dumping log to /tmp/lustre-log.1644342584.10826 [ 934.792356] Lustre: ls1-MDT0000: haven't heard from client 7c0fb59b-2290-7aef-aa30-11205caf7a7c (at 192.168.133.53@o2ib39) in 227 seconds. I think it's dead, and I am evicting it. exp ffff97f275725800, cur 1644342591 expire 1644342441 last 1644342364 [ 934.817590] Lustre: Skipped 1 previous similar message [ 937.062926] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [mdt00_008:17320] [ 937.076927] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_mgs_0009:8655] [ 937.071959] Modules linked in: osp(OE) mdd(OE) [ 937.076927] Modules linked in: [ 937.076928] osp(OE) [ 937.076929] mdd(OE) [ 937.076929] lod(OE) [ 937.076930] mdt(OE) [ 937.076930] lfsck(OE) [ 937.076931] mgs(OE) [ 937.076931] mgc(OE) [ 937.076932] osd_zfs(OE) [ 937.076932] lquota(OE) [ 937.076933] fid(OE) [ 937.076933] fld(OE) [ 937.076934] ptlrpc(OE) [ 937.076934] obdclass(OE) [ 937.076935] ko2iblnd(OE) [ 937.076935] lnet(OE) [ 937.076935] libcfs(OE) [ 937.076936] iTCO_wdt [ 937.076936] iTCO_vendor_support [ 937.076937] sb_edac [ 937.076937] intel_powerclamp [ 937.076938] coretemp [ 937.076938] intel_rapl [ 937.076939] iosf_mbi [ 937.076939] rpcrdma [ 937.076940] ib_iser [ 937.076940] kvm [ 937.076941] irqbypass [ 937.076941] joydev [ 937.076942] pcspkr [ 937.076942] lpc_ich [ 937.076943] i2c_i801 [ 937.076943] ioatdma [ 937.076943] ses [ 937.076944] enclosure [ 937.076945] sg [ 937.076945] ipmi_si [ 937.076946] ipmi_devintf [ 937.076946] ipmi_msghandler [ 937.076947] acpi_power_meter [ 937.076947] dm_round_robin [ 937.076948] acpi_cpufreq [ 937.076948] ib_ipoib [ 937.076949] rdma_ucm [ 937.076949] ib_umad [ 937.076950] sch_fq_codel [ 937.076950] iw_cxgb4 [ 937.076951] rdma_cm [ 937.076951] iw_cm [ 937.076952] ib_cm [ 937.076952] iw_cxgb3 [ 937.076952] zfs(POE) [ 937.076953] binfmt_misc [ 937.076954] zunicode(POE) [ 937.076954] zavl(POE) [ 937.076954] icp(POE) [ 937.076955] zcommon(POE) [ 937.076955] znvpair(POE) [ 937.076956] spl(OE) [ 937.076956] msr_safe(OE) [ 937.076957] ip_tables [ 937.076957] nfsv3 [ 937.076958] nfs_acl [ 937.076958] rpcsec_gss_krb5 [ 937.076959] auth_rpcgss [ 937.076960] nfsv4 [ 937.076960] dns_resolver [ 937.076960] nfs [ 937.076961] lockd [ 937.076961] grace [ 937.076962] fscache [ 937.076962] overlay(T) [ 937.076963] ext4 [ 937.076963] mbcache [ 937.076964] jbd2 [ 937.076964] dm_service_time [ 937.076965] sd_mod [ 937.076965] crc_t10dif [ 937.076966] crct10dif_generic [ 937.076966] mlx5_ib [ 937.076967] ib_uverbs [ 937.076967] ib_core [ 937.076968] be2iscsi [ 937.076968] bnx2i [ 937.076969] cnic [ 937.076969] uio [ 937.076970] cxgb4i [ 937.076970] cxgb4 [ 937.076970] cxgb3i [ 937.076971] cxgb3 [ 937.076971] mdio [ 937.076972] libcxgbi [ 937.076972] libcxgb [ 937.076973] qla4xxx [ 937.076973] iscsi_boot_sysfs [ 937.076974] mgag200 [ 937.076974] i2c_algo_bit [ 937.076975] 8021q [ 937.076975] drm_kms_helper [ 937.076976] garp [ 937.076976] syscopyarea [ 937.076977] mrp [ 937.076977] crct10dif_pclmul [ 937.076977] stp [ 937.076978] sysfillrect [ 937.076978] crct10dif_common [ 937.076979] sysimgblt [ 937.076979] crc32_pclmul [ 937.076979] llc [ 937.076980] fb_sys_fops [ 937.076980] crc32c_intel [ 937.076981] ttm [ 937.076981] ghash_clmulni_intel [ 937.076982] mlx5_core [ 937.076982] drm [ 937.076983] mxm_wmi [ 937.076983] ahci [ 937.076984] ixgbe(OE) [ 937.076984] aesni_intel [ 937.076985] mpt3sas [ 937.076985] mlxfw [ 937.076986] dm_multipath [ 937.076986] lrw [ 937.076987] devlink [ 937.076987] libahci [ 937.076988] gf128mul [ 937.076988] dca [ 937.076989] glue_helper [ 937.076989] ablk_helper [ 937.076989] raid_class [ 937.076990] ptp [ 937.076990] drm_panel_orientation_quirks [ 937.076991] libata [ 937.076991] cryptd [ 937.076992] scsi_transport_sas [ 937.076992] pps_core [ 937.076993] wmi [ 937.076993] sunrpc [ 937.076994] dm_mirror [ 937.076994] dm_region_hash [ 937.076995] dm_log [ 937.076995] dm_mod [ 937.076996] iscsi_tcp [ 937.076996] libiscsi_tcp [ 937.076997] libiscsi [ 937.076997] scsi_transport_iscsi [ 937.076998] fuse [ 937.076998] [ 937.077000] CPU: 3 PID: 8655 Comm: ll_mgs_0009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 937.077001] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 937.077002] task: ffff983229036300 ti: ffff98322904c000 task.ti: ffff98322904c000 [ 937.077003] RIP: 0010:[] [ 937.077007] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 937.077008] RSP: 0018:ffff98322904fa50 EFLAGS: 00000246 [ 937.077009] RAX: 0000000000000000 RBX: ffff97f2bfac0000 RCX: 0000000000190000 [ 937.077009] RDX: ffff97f2bfddb8c0 RSI: 0000000000b90000 RDI: ffff97f2a83bcf40 [ 937.077010] RBP: ffff98322904fa50 R08: ffff97f2bfadb8c0 R09: 0000000000000000 [ 937.077011] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98322904fa18 [ 937.077011] R13: ffffffffa00ec9b1 R14: ffff98322904f9f8 R15: ffff98322904fa28 [ 937.077012] FS: 0000000000000000(0000) GS:ffff97f2bfac0000(0000) knlGS:0000000000000000 [ 937.077013] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 937.077013] CR2: 00007ffff579f624 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 937.077014] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 937.077015] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 937.077015] Call Trace: [ 937.077021] [] queued_spin_lock_slowpath+0xb/0xf [ 937.077024] [] _raw_spin_lock+0x30/0x40 [ 937.077038] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 937.077042] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 937.077056] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 937.077058] [] ? wake_up_atomic_t+0x30/0x30 [ 937.077064] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 937.077111] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 937.077128] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 937.077134] [] ? lnet_ptl_attach_md+0x38e/0x540 [lnet] [ 937.077136] [] ? enqueue_task_fair+0x208/0x6c0 [ 937.077139] [] ? sched_clock_cpu+0xa5/0xe0 [ 937.077141] [] ? check_preempt_curr+0x80/0xa0 [ 937.077142] [] ? ttwu_do_wakeup+0x19/0x100 [ 937.077174] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 937.077196] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 937.077198] [] ? __getnstimeofday64+0x3f/0xd0 [ 937.077218] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 937.077236] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 937.077239] [] ? __wake_up_common_lock+0x91/0xc0 [ 937.077240] [] ? sched_feat_set+0xf0/0xf0 [ 937.077259] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 937.077261] [] ? __switch_to+0xce/0x5a0 [ 937.077280] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 937.077281] [] kthread+0xd1/0xe0 [ 937.077283] [] ? insert_kthread_work+0x40/0x40 [ 937.077285] [] ret_from_fork_nospec_begin+0x21/0x21 [ 937.077286] [] ? insert_kthread_work+0x40/0x40 [ 937.077287] Code: [ 937.077288] 0d [ 937.077288] 48 [ 937.077288] 98 [ 937.077288] 83 [ 937.077289] e2 [ 937.077289] 30 [ 937.077289] 48 [ 937.077289] 81 [ 937.077290] c2 [ 937.077290] c0 [ 937.077290] b8 [ 937.077290] 01 [ 937.077291] 00 [ 937.077291] 48 [ 937.077291] 03 [ 937.077291] 14 [ 937.077292] c5 [ 937.077292] e0 [ 937.077292] 17 [ 937.077292] d5 [ 937.077293] a0 [ 937.077293] 4c [ 937.077293] 89 [ 937.077293] 02 [ 937.077294] 41 [ 937.077294] 8b [ 937.077294] 40 [ 937.077294] 08 [ 937.077295] 85 [ 937.077295] c0 [ 937.077295] 75 [ 937.077295] 0f [ 937.077296] 0f [ 937.077296] 1f [ 937.077296] 44 [ 937.077296] 00 [ 937.077297] 00 [ 937.077297] f3 [ 937.077297] 90 [ 937.077297] 41 [ 937.077298] 8b [ 937.077298] 40 [ 937.077298] 08 [ 937.077299] <85> [ 937.077299] c0 [ 937.077299] 74 [ 937.077300] f6 [ 937.077300] 4d [ 937.077300] 8b [ 937.077301] 08 [ 937.077301] 4d [ 937.077301] 85 [ 937.077301] c9 [ 937.077302] 74 [ 937.077302] 04 [ 937.077302] 41 [ 937.077302] 0f [ 937.077303] 18 [ 937.077303] 09 [ 937.077303] 8b [ 937.077303] 17 [ 937.077303] 0f [ 937.077304] b7 [ 937.077304] c2 [ 937.077304] [ 937.084930] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [mdt00_002:10663] [ 937.084931] Modules linked in: [ 937.084932] osp(OE) [ 937.084932] mdd(OE) [ 937.084933] lod(OE) [ 937.084933] mdt(OE) [ 937.084933] lfsck(OE) [ 937.084933] mgs(OE) [ 937.084934] mgc(OE) [ 937.084934] osd_zfs(OE) [ 937.084934] lquota(OE) [ 937.084935] fid(OE) [ 937.084935] fld(OE) [ 937.084935] ptlrpc(OE) [ 937.084936] obdclass(OE) [ 937.084936] ko2iblnd(OE) [ 937.084936] lnet(OE) [ 937.084937] libcfs(OE) [ 937.084937] iTCO_wdt [ 937.084937] iTCO_vendor_support [ 937.084937] sb_edac [ 937.084938] intel_powerclamp [ 937.084938] coretemp [ 937.084938] intel_rapl [ 937.084939] iosf_mbi [ 937.084939] rpcrdma [ 937.084939] ib_iser [ 937.084940] kvm [ 937.084940] irqbypass [ 937.084940] joydev [ 937.084940] pcspkr [ 937.084941] lpc_ich [ 937.084941] i2c_i801 [ 937.084941] ioatdma [ 937.084941] ses [ 937.084942] enclosure [ 937.084942] sg [ 937.084942] ipmi_si [ 937.084943] ipmi_devintf [ 937.084943] ipmi_msghandler [ 937.084943] acpi_power_meter [ 937.084944] dm_round_robin [ 937.084944] acpi_cpufreq [ 937.084944] ib_ipoib [ 937.084944] rdma_ucm [ 937.084945] ib_umad [ 937.084945] sch_fq_codel [ 937.084945] iw_cxgb4 [ 937.084946] rdma_cm [ 937.084946] iw_cm [ 937.084946] ib_cm [ 937.084946] iw_cxgb3 [ 937.084947] zfs(POE) [ 937.084947] binfmt_misc [ 937.084947] zunicode(POE) [ 937.084948] zavl(POE) [ 937.084948] icp(POE) [ 937.084948] zcommon(POE) [ 937.084949] znvpair(POE) [ 937.084949] spl(OE) [ 937.084949] msr_safe(OE) [ 937.084950] ip_tables [ 937.084950] nfsv3 [ 937.084950] nfs_acl [ 937.084951] rpcsec_gss_krb5 [ 937.084951] auth_rpcgss [ 937.084951] nfsv4 [ 937.084951] dns_resolver [ 937.084952] nfs [ 937.084952] lockd [ 937.084952] grace [ 937.084953] fscache [ 937.084953] overlay(T) [ 937.084953] ext4 [ 937.084953] mbcache [ 937.084954] jbd2 [ 937.084954] dm_service_time [ 937.084954] sd_mod [ 937.084955] crc_t10dif [ 937.084955] crct10dif_generic [ 937.084955] mlx5_ib [ 937.084955] ib_uverbs [ 937.084956] ib_core [ 937.084956] be2iscsi [ 937.084956] bnx2i [ 937.084957] cnic [ 937.084957] uio [ 937.084957] cxgb4i [ 937.084957] cxgb4 [ 937.084958] cxgb3i [ 937.084958] cxgb3 [ 937.084958] mdio [ 937.084959] libcxgbi [ 937.084959] libcxgb [ 937.084959] qla4xxx [ 937.084959] iscsi_boot_sysfs [ 937.084960] mgag200 [ 937.084960] i2c_algo_bit [ 937.084960] 8021q [ 937.084960] drm_kms_helper [ 937.084961] garp [ 937.084961] syscopyarea [ 937.084961] mrp [ 937.084962] crct10dif_pclmul [ 937.084962] stp [ 937.084962] sysfillrect [ 937.084962] crct10dif_common [ 937.084963] sysimgblt [ 937.084963] crc32_pclmul [ 937.084963] llc [ 937.084964] fb_sys_fops [ 937.084964] crc32c_intel [ 937.084964] ttm [ 937.084964] ghash_clmulni_intel [ 937.084965] mlx5_core [ 937.084965] drm [ 937.084965] mxm_wmi [ 937.084966] ahci [ 937.084966] ixgbe(OE) [ 937.084966] aesni_intel [ 937.084966] mpt3sas [ 937.084967] mlxfw [ 937.084967] dm_multipath [ 937.084967] lrw [ 937.084968] devlink [ 937.084968] libahci [ 937.084968] gf128mul [ 937.084968] dca [ 937.084969] glue_helper [ 937.084969] ablk_helper [ 937.084969] raid_class [ 937.084969] ptp [ 937.084970] drm_panel_orientation_quirks [ 937.084970] libata [ 937.084970] cryptd [ 937.084971] scsi_transport_sas [ 937.084971] pps_core [ 937.084971] wmi [ 937.084971] sunrpc [ 937.084972] dm_mirror [ 937.084972] dm_region_hash [ 937.084972] dm_log [ 937.084973] dm_mod [ 937.084973] iscsi_tcp [ 937.084973] libiscsi_tcp [ 937.084973] libiscsi [ 937.084974] scsi_transport_iscsi [ 937.084974] fuse [ 937.084974] [ 937.084976] CPU: 5 PID: 10663 Comm: mdt00_002 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 937.084976] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 937.084977] task: ffff97f28e72a100 ti: ffff97f28e600000 task.ti: ffff97f28e600000 [ 937.084978] RIP: 0010:[] [ 937.084980] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 937.084980] RSP: 0018:ffff97f28e603ab0 EFLAGS: 00000246 [ 937.084981] RAX: 0000000000000000 RBX: ffff97f2bfb40000 RCX: 0000000000290000 [ 937.084982] RDX: ffff9832bf21b8c0 RSI: 0000000000410001 RDI: ffff97f2a83bcf40 [ 937.084982] RBP: ffff97f28e603ab0 R08: ffff97f2bfb5b8c0 R09: 0000000000000000 [ 937.084983] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f28e603a78 [ 937.084984] R13: ffffffffa00ec9b1 R14: ffff97f28e603a58 R15: ffff97f28e603a88 [ 937.084985] FS: 0000000000000000(0000) GS:ffff97f2bfb40000(0000) knlGS:0000000000000000 [ 937.084985] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 937.084986] CR2: 00007ffff1096160 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 937.084987] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 937.084987] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 937.084988] Call Trace: [ 937.084990] [] queued_spin_lock_slowpath+0xb/0xf [ 937.084992] [] _raw_spin_lock+0x30/0x40 [ 937.084997] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 937.085001] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 937.085008] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 937.085009] [] ? wake_up_atomic_t+0x30/0x30 [ 937.085015] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 937.085034] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 937.085052] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 937.085071] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 937.085089] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 937.085106] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 937.085129] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 937.085150] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 937.085151] [] ? __getnstimeofday64+0x3f/0xd0 [ 937.085177] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 937.085202] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 937.085203] [] ? __wake_up_common_lock+0x91/0xc0 [ 937.085204] [] ? sched_feat_set+0xf0/0xf0 [ 937.085230] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 937.085231] [] ? __switch_to+0xce/0x5a0 [ 937.085257] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 937.085258] [] kthread+0xd1/0xe0 [ 937.085260] [] ? insert_kthread_work+0x40/0x40 [ 937.085261] [] ret_from_fork_nospec_begin+0x21/0x21 [ 937.085263] [] ? insert_kthread_work+0x40/0x40 [ 937.085263] Code: [ 937.085264] 0d [ 937.085264] 48 [ 937.085264] 98 [ 937.085265] 83 [ 937.085265] e2 [ 937.085265] 30 [ 937.085265] 48 [ 937.085265] 81 [ 937.085266] c2 [ 937.085266] c0 [ 937.085266] b8 [ 937.085266] 01 [ 937.085267] 00 [ 937.085267] 48 [ 937.085267] 03 [ 937.085267] 14 [ 937.085268] c5 [ 937.085268] e0 [ 937.085268] 17 [ 937.085268] d5 [ 937.085269] a0 [ 937.085269] 4c [ 937.085269] 89 [ 937.085270] 02 [ 937.085270] 41 [ 937.085270] 8b [ 937.085270] 40 [ 937.085271] 08 [ 937.085271] 85 [ 937.085271] c0 [ 937.085271] 75 [ 937.085272] 0f [ 937.085272] 0f [ 937.085272] 1f [ 937.085272] 44 [ 937.085273] 00 [ 937.085273] 00 [ 937.085273] f3 [ 937.085273] 90 [ 937.085274] 41 [ 937.085274] 8b [ 937.085274] 40 [ 937.085275] 08 [ 937.085275] <85> [ 937.085275] c0 [ 937.085276] 74 [ 937.085276] f6 [ 937.085276] 4d [ 937.085276] 8b [ 937.085277] 08 [ 937.085277] 4d [ 937.085277] 85 [ 937.085277] c9 [ 937.085278] 74 [ 937.085278] 04 [ 937.085278] 41 [ 937.085278] 0f [ 937.085279] 18 [ 937.085279] 09 [ 937.085279] 8b [ 937.085279] 17 [ 937.085280] 0f [ 937.085280] b7 [ 937.085280] c2 [ 937.085280] [ 937.191942] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [mdt01_003:10825] [ 937.191942] Modules linked in: [ 937.191943] osp(OE) [ 937.191944] mdd(OE) [ 937.191944] lod(OE) [ 937.191944] mdt(OE) [ 937.191945] lfsck(OE) [ 937.191945] mgs(OE) [ 937.191946] mgc(OE) [ 937.191946] osd_zfs(OE) [ 937.191947] lquota(OE) [ 937.191947] fid(OE) [ 937.191948] fld(OE) [ 937.191948] ptlrpc(OE) [ 937.191949] obdclass(OE) [ 937.191949] ko2iblnd(OE) [ 937.191949] lnet(OE) [ 937.191950] libcfs(OE) [ 937.191950] iTCO_wdt [ 937.191951] iTCO_vendor_support [ 937.191951] sb_edac [ 937.191952] intel_powerclamp [ 937.191952] coretemp [ 937.191953] intel_rapl [ 937.191953] iosf_mbi [ 937.191954] rpcrdma [ 937.191954] ib_iser [ 937.191955] kvm [ 937.191955] irqbypass [ 937.191956] joydev [ 937.191956] pcspkr [ 937.191956] lpc_ich [ 937.191957] i2c_i801 [ 937.191957] ioatdma [ 937.191958] ses [ 937.191958] enclosure [ 937.191958] sg [ 937.191959] ipmi_si [ 937.191960] ipmi_devintf [ 937.191960] ipmi_msghandler [ 937.191960] acpi_power_meter [ 937.191961] dm_round_robin [ 937.191961] acpi_cpufreq [ 937.191962] ib_ipoib [ 937.191962] rdma_ucm [ 937.191963] ib_umad [ 937.191963] sch_fq_codel [ 937.191963] iw_cxgb4 [ 937.191964] rdma_cm [ 937.191964] iw_cm [ 937.191965] ib_cm [ 937.191965] iw_cxgb3 [ 937.191966] zfs(POE) [ 937.191966] binfmt_misc [ 937.191967] zunicode(POE) [ 937.191967] zavl(POE) [ 937.191968] icp(POE) [ 937.191968] zcommon(POE) [ 937.191969] znvpair(POE) [ 937.191969] spl(OE) [ 937.191970] msr_safe(OE) [ 937.191970] ip_tables [ 937.191971] nfsv3 [ 937.191971] nfs_acl [ 937.191972] rpcsec_gss_krb5 [ 937.191972] auth_rpcgss [ 937.191973] nfsv4 [ 937.191973] dns_resolver [ 937.191973] nfs [ 937.191974] lockd [ 937.191974] grace [ 937.191975] fscache [ 937.191975] overlay(T) [ 937.191976] ext4 [ 937.191976] mbcache [ 937.191977] jbd2 [ 937.191977] dm_service_time [ 937.191978] sd_mod [ 937.191978] crc_t10dif [ 937.191979] crct10dif_generic [ 937.191979] mlx5_ib [ 937.191980] ib_uverbs [ 937.191980] ib_core [ 937.191980] be2iscsi [ 937.191981] bnx2i [ 937.191981] cnic [ 937.191981] uio [ 937.191982] cxgb4i [ 937.191983] cxgb4 [ 937.191983] cxgb3i [ 937.191983] cxgb3 [ 937.191984] mdio [ 937.191985] libcxgbi [ 937.191985] libcxgb [ 937.191986] qla4xxx [ 937.191986] iscsi_boot_sysfs [ 937.191986] mgag200 [ 937.191987] i2c_algo_bit [ 937.191987] 8021q [ 937.191988] drm_kms_helper [ 937.191988] garp [ 937.191989] syscopyarea [ 937.191989] mrp [ 937.191990] crct10dif_pclmul [ 937.191990] stp [ 937.191991] sysfillrect [ 937.191991] crct10dif_common [ 937.191992] sysimgblt [ 937.191992] crc32_pclmul [ 937.191993] llc [ 937.191993] fb_sys_fops [ 937.191994] crc32c_intel [ 937.191994] ttm [ 937.191995] ghash_clmulni_intel [ 937.191995] mlx5_core [ 937.191996] drm [ 937.191996] mxm_wmi [ 937.191996] ahci [ 937.191997] ixgbe(OE) [ 937.191997] aesni_intel [ 937.191998] mpt3sas [ 937.191998] mlxfw [ 937.191999] dm_multipath [ 937.191999] lrw [ 937.191999] devlink [ 937.192000] libahci [ 937.192000] gf128mul [ 937.192001] dca [ 937.192001] glue_helper [ 937.192002] ablk_helper [ 937.192002] raid_class [ 937.192003] ptp [ 937.192003] drm_panel_orientation_quirks [ 937.192004] libata [ 937.192004] cryptd [ 937.192005] scsi_transport_sas [ 937.192005] pps_core [ 937.192005] wmi [ 937.192006] sunrpc [ 937.192006] dm_mirror [ 937.192007] dm_region_hash [ 937.192007] dm_log [ 937.192008] dm_mod [ 937.192008] iscsi_tcp [ 937.192009] libiscsi_tcp [ 937.192009] libiscsi [ 937.192010] scsi_transport_iscsi [ 937.192010] fuse [ 937.192010] [ 937.192013] CPU: 13 PID: 10825 Comm: mdt01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 937.192014] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 937.192015] task: ffff98322385d280 ti: ffff983225500000 task.ti: ffff983225500000 [ 937.192016] RIP: 0010:[] [ 937.192021] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 937.192022] RSP: 0018:ffff983225503a50 EFLAGS: 00000246 [ 937.192022] RAX: 0000000000000000 RBX: ffff9832bf340000 RCX: 0000000000690000 [ 937.192023] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 937.192024] RBP: ffff983225503a50 R08: ffff9832bf35b8c0 R09: 0000000000000000 [ 937.192025] R10: 0000000000000001 R11: 0000000000000005 R12: ffff983225503a18 [ 937.192026] R13: ffffffffa00ec9b1 R14: ffff9832255039f8 R15: ffff983225503a28 [ 937.192027] FS: 0000000000000000(0000) GS:ffff9832bf340000(0000) knlGS:0000000000000000 [ 937.192028] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 937.192029] CR2: 00002aaaab176a00 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 937.192031] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 937.192031] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 937.192032] Call Trace: [ 937.192036] [] queued_spin_lock_slowpath+0xb/0xf [ 937.192039] [] _raw_spin_lock+0x30/0x40 [ 937.192050] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 937.192070] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 937.192073] [] ? wake_up_atomic_t+0x30/0x30 [ 937.192080] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 937.192116] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 937.192138] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 937.192141] [] ? enqueue_task_fair+0x208/0x6c0 [ 937.192143] [] ? check_preempt_curr+0x80/0xa0 [ 937.192145] [] ? ttwu_do_wakeup+0x19/0x100 [ 937.192180] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 937.192208] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 937.192210] [] ? __getnstimeofday64+0x3f/0xd0 [ 937.192236] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 937.192261] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 937.192262] [] ? __wake_up_common_lock+0x91/0xc0 [ 937.192264] [] ? sched_feat_set+0xf0/0xf0 [ 937.192288] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 937.192313] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 937.192315] [] kthread+0xd1/0xe0 [ 937.192317] [] ? insert_kthread_work+0x40/0x40 [ 937.192319] [] ret_from_fork_nospec_begin+0x21/0x21 [ 937.192321] [] ? insert_kthread_work+0x40/0x40 [ 937.192321] Code: [ 937.192322] 13 [ 937.192322] 48 [ 937.192323] c1 [ 937.192323] ea [ 937.192323] 0d [ 937.192324] 48 [ 937.192324] 98 [ 937.192324] 83 [ 937.192325] e2 [ 937.192325] 30 [ 937.192325] 48 [ 937.192326] 81 [ 937.192326] c2 [ 937.192326] c0 [ 937.192327] b8 [ 937.192327] 01 [ 937.192327] 00 [ 937.192328] 48 [ 937.192328] 03 [ 937.192329] 14 [ 937.192329] c5 [ 937.192329] e0 [ 937.192330] 17 [ 937.192330] d5 [ 937.192330] a0 [ 937.192331] 4c [ 937.192331] 89 [ 937.192332] 02 [ 937.192332] 41 [ 937.192333] 8b [ 937.192333] 40 [ 937.192333] 08 [ 937.192334] 85 [ 937.192334] c0 [ 937.192334] 75 [ 937.192335] 0f [ 937.192335] 0f [ 937.192336] 1f [ 937.192336] 44 [ 937.192336] 00 [ 937.192337] 00 [ 937.192337] f3 [ 937.192338] 90 [ 937.192338] <41> [ 937.192338] 8b [ 937.192339] 40 [ 937.192339] 08 [ 937.192340] 85 [ 937.192340] c0 [ 937.192340] 74 [ 937.192341] f6 [ 937.192341] 4d [ 937.192341] 8b [ 937.192342] 08 [ 937.192342] 4d [ 937.192343] 85 [ 937.192343] c9 [ 937.192343] 74 [ 937.192344] 04 [ 937.192344] 41 [ 937.192344] 0f [ 937.192345] 18 [ 937.192345] 09 [ 937.192346] 8b [ 937.192346] [ 938.610020] lod(OE) [ 938.612950] mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 [ 938.691123] cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 938.747742] CPU: 0 PID: 17320 Comm: mdt00_008 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 938.762009] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 938.774337] task: ffff98321989e300 ti: ffff98321cd1c000 task.ti: ffff98321cd1c000 [ 938.783171] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 938.794734] RSP: 0018:ffff98321cd1fab0 EFLAGS: 00000246 [ 938.801145] RAX: 0000000000000000 RBX: ffff97f2bfa00000 RCX: 0000000000010000 [ 938.809592] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 938.818039] RBP: ffff98321cd1fab0 R08: ffff97f2bfa1b8c0 R09: 0000000000000000 [ 938.826488] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321cd1fa78 [ 938.834935] R13: ffffffffa00ec9b1 R14: ffff98321cd1fa58 R15: ffff98321cd1fa88 [ 938.843382] FS: 0000000000000000(0000) GS:ffff97f2bfa00000(0000) knlGS:0000000000000000 [ 938.852895] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 938.859791] CR2: 00007ffff7ff8000 CR3: 0000007f0e586000 CR4: 00000000003607f0 [ 938.868238] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 938.876685] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 938.885131] Call Trace: [ 938.888343] [] queued_spin_lock_slowpath+0xb/0xf [ 938.895822] [] _raw_spin_lock+0x30/0x40 [ 938.902433] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 938.910114] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 938.918563] [] ? wake_up_atomic_t+0x30/0x30 [ 938.925562] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 938.932967] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 938.941245] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 938.949209] [] ? check_preempt_curr+0x80/0xa0 [ 938.956419] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 938.964308] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 938.971622] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 938.979806] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 938.988739] [] ? __getnstimeofday64+0x3f/0xd0 [ 938.995951] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 939.005004] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 939.013065] [] ? __wake_up_common_lock+0x91/0xc0 [ 939.020543] [] ? sched_feat_set+0xf0/0xf0 [ 939.027367] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 939.034849] [] ? __switch_to+0xce/0x5a0 [ 939.041480] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 939.050219] [] kthread+0xd1/0xe0 [ 939.056146] [] ? insert_kthread_work+0x40/0x40 [ 939.063430] [] ret_from_fork_nospec_begin+0x21/0x21 [ 939.071199] [] ? insert_kthread_work+0x40/0x40 [ 939.078483] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 945.219863] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [mdt00_009:17321] [ 945.228990] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 945.308798] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 945.369852] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 945.384215] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 945.396541] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 945.405377] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 945.416938] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 945.423349] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 945.431796] RDX: ffff9832bf55b8c0 RSI: 0000000000e90000 RDI: ffff97f2a83bcf40 [ 945.440242] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 945.448689] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 945.457137] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 945.465584] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 945.475099] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 945.481995] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 945.490444] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 945.498891] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 945.507338] Call Trace: [ 945.510549] [] queued_spin_lock_slowpath+0xb/0xf [ 945.518029] [] _raw_spin_lock+0x30/0x40 [ 945.524642] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 945.532321] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 945.540770] [] ? wake_up_atomic_t+0x30/0x30 [ 945.547769] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 945.555182] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 945.563455] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 945.572016] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 945.580175] [] ? enqueue_task_fair+0x208/0x6c0 [ 945.587462] [] ? sched_clock_cpu+0xa5/0xe0 [ 945.594359] [] ? check_preempt_curr+0x80/0xa0 [ 945.601545] [] ? ttwu_do_wakeup+0x19/0x100 [ 945.608466] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 945.616644] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 945.625570] [] ? __getnstimeofday64+0x3f/0xd0 [ 945.632776] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 945.641822] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 945.649883] [] ? __wake_up_common_lock+0x91/0xc0 [ 945.657361] [] ? sched_feat_set+0xf0/0xf0 [ 945.664179] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 945.671659] [] ? __switch_to+0xce/0x5a0 [ 945.678284] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 945.687022] [] kthread+0xd1/0xe0 [ 945.692950] [] ? insert_kthread_work+0x40/0x40 [ 945.700235] [] ret_from_fork_nospec_begin+0x21/0x21 [ 945.708004] [] ? insert_kthread_work+0x40/0x40 [ 945.715287] Code: c1 e8 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 [ 946.737084] Lustre: 7366:0:(client.c:2169:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1644342464/real 1644342464] req@ffff97f25e68c380 x1724217557263872/t0(0) o400->ls1-OST001d-osc-MDT0000@172.19.3.143@o2ib600:28/4 lens 224/224 e 0 to 1 dl 1644342602 ref 1 fl Rpc:X/c0/ffffffff rc 0/-1 [ 946.769494] Lustre: 7366:0:(client.c:2169:ptlrpc_expire_one_request()) Skipped 66 previous similar messages [ 949.074307] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [mdt00_013:17594] [ 949.083338] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 949.171321] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [mdt01_022:17601] [ 949.163181] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx [ 949.171322] Modules linked in: [ 949.171324] osp(OE) [ 949.171324] mdd(OE) [ 949.171325] lod(OE) [ 949.171326] mdt(OE) [ 949.171326] lfsck(OE) [ 949.171327] mgs(OE) [ 949.171327] mgc(OE) [ 949.171327] osd_zfs(OE) [ 949.171328] lquota(OE) [ 949.171328] fid(OE) [ 949.171329] fld(OE) [ 949.171329] ptlrpc(OE) [ 949.171330] obdclass(OE) [ 949.171330] ko2iblnd(OE) [ 949.171330] lnet(OE) [ 949.171331] libcfs(OE) [ 949.171331] iTCO_wdt [ 949.171332] iTCO_vendor_support [ 949.171332] sb_edac [ 949.171333] intel_powerclamp [ 949.171333] coretemp [ 949.171334] intel_rapl [ 949.171334] iosf_mbi [ 949.171334] rpcrdma [ 949.171335] ib_iser [ 949.171336] kvm [ 949.171336] irqbypass [ 949.171337] joydev [ 949.171337] pcspkr [ 949.171338] lpc_ich [ 949.171338] i2c_i801 [ 949.171339] ioatdma [ 949.171339] ses [ 949.171340] enclosure [ 949.171340] sg [ 949.171340] ipmi_si [ 949.171341] ipmi_devintf [ 949.171341] ipmi_msghandler [ 949.171342] acpi_power_meter [ 949.171343] dm_round_robin [ 949.171343] acpi_cpufreq [ 949.171344] ib_ipoib [ 949.171344] rdma_ucm [ 949.171345] ib_umad [ 949.171345] sch_fq_codel [ 949.171345] iw_cxgb4 [ 949.171346] rdma_cm [ 949.171346] iw_cm [ 949.171347] ib_cm [ 949.171347] iw_cxgb3 [ 949.171348] zfs(POE) [ 949.171349] binfmt_misc [ 949.171349] zunicode(POE) [ 949.171350] zavl(POE) [ 949.171350] icp(POE) [ 949.171351] zcommon(POE) [ 949.171351] znvpair(POE) [ 949.171351] spl(OE) [ 949.171352] msr_safe(OE) [ 949.171352] ip_tables [ 949.171353] nfsv3 [ 949.171353] nfs_acl [ 949.171354] rpcsec_gss_krb5 [ 949.171354] auth_rpcgss [ 949.171355] nfsv4 [ 949.171355] dns_resolver [ 949.171356] nfs [ 949.171356] lockd [ 949.171357] grace [ 949.171357] fscache [ 949.171358] overlay(T) [ 949.171358] ext4 [ 949.171359] mbcache [ 949.171359] jbd2 [ 949.171360] dm_service_time [ 949.171360] sd_mod [ 949.171361] crc_t10dif [ 949.171361] crct10dif_generic [ 949.171362] mlx5_ib [ 949.171362] ib_uverbs [ 949.171363] ib_core [ 949.171363] be2iscsi [ 949.171364] bnx2i [ 949.171364] cnic [ 949.171364] uio [ 949.171365] cxgb4i [ 949.171365] cxgb4 [ 949.171366] cxgb3i [ 949.171366] cxgb3 [ 949.171367] mdio [ 949.171367] libcxgbi [ 949.171368] libcxgb [ 949.171368] qla4xxx [ 949.171368] iscsi_boot_sysfs [ 949.171369] mgag200 [ 949.171369] i2c_algo_bit [ 949.171370] 8021q [ 949.171370] drm_kms_helper [ 949.171371] garp [ 949.171371] syscopyarea [ 949.171372] mrp [ 949.171373] crct10dif_pclmul [ 949.171373] stp [ 949.171373] sysfillrect [ 949.171374] crct10dif_common [ 949.171374] sysimgblt [ 949.171375] crc32_pclmul [ 949.171375] llc [ 949.171376] fb_sys_fops [ 949.171376] crc32c_intel [ 949.171377] ttm [ 949.171377] ghash_clmulni_intel [ 949.171377] mlx5_core [ 949.171378] drm [ 949.171378] mxm_wmi [ 949.171379] ahci [ 949.171379] ixgbe(OE) [ 949.171380] aesni_intel [ 949.171380] mpt3sas [ 949.171380] mlxfw [ 949.171381] dm_multipath [ 949.171381] lrw [ 949.171382] devlink [ 949.171382] libahci [ 949.171382] gf128mul [ 949.171383] dca [ 949.171383] glue_helper [ 949.171383] ablk_helper [ 949.171384] raid_class [ 949.171384] ptp [ 949.171385] drm_panel_orientation_quirks [ 949.171385] libata [ 949.171386] cryptd [ 949.171386] scsi_transport_sas [ 949.171387] pps_core [ 949.171387] wmi [ 949.171387] sunrpc [ 949.171388] dm_mirror [ 949.171388] dm_region_hash [ 949.171388] dm_log [ 949.171389] dm_mod [ 949.171389] iscsi_tcp [ 949.171390] libiscsi_tcp [ 949.171390] libiscsi [ 949.171391] scsi_transport_iscsi [ 949.171391] fuse [ 949.171391] [ 949.171394] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.171394] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.171395] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 949.171396] RIP: 0010:[] [ 949.171403] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 949.171404] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 949.171404] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 949.171405] RDX: ffff9832bf35b8c0 RSI: 0000000000690001 RDI: ffff97f2a83bcf40 [ 949.171406] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 949.171407] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 949.171407] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 949.171408] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 949.171409] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.171410] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 949.171411] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.171411] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.171412] Call Trace: [ 949.171418] [] queued_spin_lock_slowpath+0xb/0xf [ 949.171422] [] _raw_spin_lock+0x30/0x40 [ 949.171433] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.171453] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.171456] [] ? wake_up_atomic_t+0x30/0x30 [ 949.171462] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.171500] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.171519] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 949.171538] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 949.171556] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 949.171573] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 949.171601] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 949.171622] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.171624] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.171643] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.171661] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.171663] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.171665] [] ? sched_feat_set+0xf0/0xf0 [ 949.171683] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.171701] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.171703] [] kthread+0xd1/0xe0 [ 949.171704] [] ? insert_kthread_work+0x40/0x40 [ 949.171706] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.171708] [] ? insert_kthread_work+0x40/0x40 [ 949.171708] Code: [ 949.171709] 13 [ 949.171709] 48 [ 949.171709] c1 [ 949.171709] ea [ 949.171710] 0d [ 949.171710] 48 [ 949.171710] 98 [ 949.171711] 83 [ 949.171711] e2 [ 949.171711] 30 [ 949.171711] 48 [ 949.171711] 81 [ 949.171712] c2 [ 949.171712] c0 [ 949.171712] b8 [ 949.171713] 01 [ 949.171713] 00 [ 949.171713] 48 [ 949.171713] 03 [ 949.171714] 14 [ 949.171714] c5 [ 949.171714] e0 [ 949.171714] 17 [ 949.171715] d5 [ 949.171715] a0 [ 949.171715] 4c [ 949.171715] 89 [ 949.171716] 02 [ 949.171716] 41 [ 949.171716] 8b [ 949.171717] 40 [ 949.171717] 08 [ 949.171717] 85 [ 949.171718] c0 [ 949.171718] 75 [ 949.171718] 0f [ 949.171718] 0f [ 949.171719] 1f [ 949.171719] 44 [ 949.171719] 00 [ 949.171719] 00 [ 949.171720] f3 [ 949.171720] 90 [ 949.171720] <41> [ 949.171720] 8b [ 949.171721] 40 [ 949.171721] 08 [ 949.171721] 85 [ 949.171722] c0 [ 949.171722] 74 [ 949.171722] f6 [ 949.171722] 4d [ 949.171723] 8b [ 949.171723] 08 [ 949.171723] 4d [ 949.171723] 85 [ 949.171724] c9 [ 949.171724] 74 [ 949.171724] 04 [ 949.171725] 41 [ 949.171725] 0f [ 949.171725] 18 [ 949.171725] 09 [ 949.171726] 8b [ 949.171726] [ 949.198323] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [mdt01_016:17554] [ 949.198324] Modules linked in: [ 949.198324] osp(OE) [ 949.198325] mdd(OE) [ 949.198325] lod(OE) [ 949.198326] mdt(OE) [ 949.198326] lfsck(OE) [ 949.198326] mgs(OE) [ 949.198327] mgc(OE) [ 949.198327] osd_zfs(OE) [ 949.198328] lquota(OE) [ 949.198328] fid(OE) [ 949.198329] fld(OE) [ 949.198329] ptlrpc(OE) [ 949.198330] obdclass(OE) [ 949.198330] ko2iblnd(OE) [ 949.198331] lnet(OE) [ 949.198331] libcfs(OE) [ 949.198331] iTCO_wdt [ 949.198332] iTCO_vendor_support [ 949.198332] sb_edac [ 949.198333] intel_powerclamp [ 949.198333] coretemp [ 949.198333] intel_rapl [ 949.198334] iosf_mbi [ 949.198334] rpcrdma [ 949.198334] ib_iser [ 949.198335] kvm [ 949.198335] irqbypass [ 949.198336] joydev [ 949.198336] pcspkr [ 949.198336] lpc_ich [ 949.198337] i2c_i801 [ 949.198337] ioatdma [ 949.198337] ses [ 949.198338] enclosure [ 949.198338] sg [ 949.198338] ipmi_si [ 949.198339] ipmi_devintf [ 949.198339] ipmi_msghandler [ 949.198340] acpi_power_meter [ 949.198340] dm_round_robin [ 949.198341] acpi_cpufreq [ 949.198341] ib_ipoib [ 949.198342] rdma_ucm [ 949.198342] ib_umad [ 949.198342] sch_fq_codel [ 949.198343] iw_cxgb4 [ 949.198343] rdma_cm [ 949.198344] iw_cm [ 949.198344] ib_cm [ 949.198344] iw_cxgb3 [ 949.198345] zfs(POE) [ 949.198345] binfmt_misc [ 949.198346] zunicode(POE) [ 949.198346] zavl(POE) [ 949.198347] icp(POE) [ 949.198347] zcommon(POE) [ 949.198348] znvpair(POE) [ 949.198348] spl(OE) [ 949.198348] msr_safe(OE) [ 949.198349] ip_tables [ 949.198349] nfsv3 [ 949.198350] nfs_acl [ 949.198350] rpcsec_gss_krb5 [ 949.198351] auth_rpcgss [ 949.198351] nfsv4 [ 949.198351] dns_resolver [ 949.198352] nfs [ 949.198352] lockd [ 949.198352] grace [ 949.198353] fscache [ 949.198353] overlay(T) [ 949.198354] ext4 [ 949.198354] mbcache [ 949.198354] jbd2 [ 949.198355] dm_service_time [ 949.198355] sd_mod [ 949.198356] crc_t10dif [ 949.198356] crct10dif_generic [ 949.198356] mlx5_ib [ 949.198357] ib_uverbs [ 949.198357] ib_core [ 949.198357] be2iscsi [ 949.198358] bnx2i [ 949.198358] cnic [ 949.198359] uio [ 949.198359] cxgb4i [ 949.198359] cxgb4 [ 949.198360] cxgb3i [ 949.198360] cxgb3 [ 949.198361] mdio [ 949.198361] libcxgbi [ 949.198361] libcxgb [ 949.198362] qla4xxx [ 949.198362] iscsi_boot_sysfs [ 949.198363] mgag200 [ 949.198363] i2c_algo_bit [ 949.198363] 8021q [ 949.198364] drm_kms_helper [ 949.198364] garp [ 949.198365] syscopyarea [ 949.198365] mrp [ 949.198365] crct10dif_pclmul [ 949.198366] stp [ 949.198366] sysfillrect [ 949.198367] crct10dif_common [ 949.198367] sysimgblt [ 949.198367] crc32_pclmul [ 949.198368] llc [ 949.198368] fb_sys_fops [ 949.198369] crc32c_intel [ 949.198369] ttm [ 949.198369] ghash_clmulni_intel [ 949.198370] mlx5_core [ 949.198370] drm [ 949.198370] mxm_wmi [ 949.198371] ahci [ 949.198371] ixgbe(OE) [ 949.198372] aesni_intel [ 949.198372] mpt3sas [ 949.198373] mlxfw [ 949.198373] dm_multipath [ 949.198373] lrw [ 949.198374] devlink [ 949.198374] libahci [ 949.198374] gf128mul [ 949.198375] dca [ 949.198375] glue_helper [ 949.198376] ablk_helper [ 949.198376] raid_class [ 949.198376] ptp [ 949.198377] drm_panel_orientation_quirks [ 949.198377] libata [ 949.198378] cryptd [ 949.198378] scsi_transport_sas [ 949.198379] pps_core [ 949.198379] wmi [ 949.198379] sunrpc [ 949.198380] dm_mirror [ 949.198380] dm_region_hash [ 949.198381] dm_log [ 949.198381] dm_mod [ 949.198381] iscsi_tcp [ 949.198382] libiscsi_tcp [ 949.198382] libiscsi [ 949.198383] scsi_transport_iscsi [ 949.198383] fuse [ 949.198383] [ 949.198385] CPU: 14 PID: 17554 Comm: mdt01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.198386] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.198387] task: ffff98321c985280 ti: ffff98321b544000 task.ti: ffff98321b544000 [ 949.198388] RIP: 0010:[] [ 949.198390] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 949.198391] RSP: 0018:ffff98321b547ab0 EFLAGS: 00000246 [ 949.198392] RAX: 0000000000000000 RBX: ffff9832bf380000 RCX: 0000000000710000 [ 949.198393] RDX: ffff97f2bfc5b8c0 RSI: 0000000000890000 RDI: ffff97f2a83bcf40 [ 949.198394] RBP: ffff98321b547ab0 R08: ffff9832bf39b8c0 R09: 0000000000000000 [ 949.198395] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321b547a78 [ 949.198396] R13: ffffffffa00ec9b1 R14: ffff98321b547a58 R15: ffff98321b547a88 [ 949.198397] FS: 0000000000000000(0000) GS:ffff9832bf380000(0000) knlGS:0000000000000000 [ 949.198398] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.198399] CR2: 00007ffff7fd6000 CR3: 0000003f67e7a000 CR4: 00000000003607e0 [ 949.198400] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.198401] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.198401] Call Trace: [ 949.198404] [] queued_spin_lock_slowpath+0xb/0xf [ 949.198405] [] _raw_spin_lock+0x30/0x40 [ 949.198412] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.198420] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.198422] [] ? wake_up_atomic_t+0x30/0x30 [ 949.198430] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.198454] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.198479] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 949.198481] [] ? check_preempt_curr+0x80/0xa0 [ 949.198504] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 949.198527] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 949.198558] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 949.198586] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.198589] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.198615] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.198640] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.198641] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.198643] [] ? sched_feat_set+0xf0/0xf0 [ 949.198667] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.198691] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.198693] [] kthread+0xd1/0xe0 [ 949.198695] [] ? insert_kthread_work+0x40/0x40 [ 949.198697] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.198699] [] ? insert_kthread_work+0x40/0x40 [ 949.198699] Code: [ 949.198700] 13 [ 949.198700] 48 [ 949.198700] c1 [ 949.198701] ea [ 949.198701] 0d [ 949.198701] 48 [ 949.198702] 98 [ 949.198702] 83 [ 949.198703] e2 [ 949.198703] 30 [ 949.198703] 48 [ 949.198704] 81 [ 949.198704] c2 [ 949.198704] c0 [ 949.198705] b8 [ 949.198705] 01 [ 949.198706] 00 [ 949.198706] 48 [ 949.198706] 03 [ 949.198707] 14 [ 949.198707] c5 [ 949.198708] e0 [ 949.198708] 17 [ 949.198708] d5 [ 949.198709] a0 [ 949.198709] 4c [ 949.198709] 89 [ 949.198710] 02 [ 949.198710] 41 [ 949.198711] 8b [ 949.198711] 40 [ 949.198711] 08 [ 949.198712] 85 [ 949.198712] c0 [ 949.198712] 75 [ 949.198713] 0f [ 949.198713] 0f [ 949.198713] 1f [ 949.198714] 44 [ 949.198714] 00 [ 949.198715] 00 [ 949.198715] f3 [ 949.198715] 90 [ 949.198716] <41> [ 949.198716] 8b [ 949.198716] 40 [ 949.198717] 08 [ 949.198717] 85 [ 949.198718] c0 [ 949.198718] 74 [ 949.198718] f6 [ 949.198719] 4d [ 949.198719] 8b [ 949.198719] 08 [ 949.198720] 4d [ 949.198720] 85 [ 949.198720] c9 [ 949.198721] 74 [ 949.198721] 04 [ 949.198722] 41 [ 949.198722] 0f [ 949.198722] 18 [ 949.198723] 09 [ 949.198723] 8b [ 949.198723] [ 949.202322] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [mdt01_007:17310] [ 949.202322] Modules linked in: [ 949.202323] osp(OE) [ 949.202324] mdd(OE) [ 949.202324] lod(OE) [ 949.202325] mdt(OE) [ 949.202325] lfsck(OE) [ 949.202326] mgs(OE) [ 949.202326] mgc(OE) [ 949.202326] osd_zfs(OE) [ 949.202326] lquota(OE) [ 949.202327] fid(OE) [ 949.202327] fld(OE) [ 949.202327] ptlrpc(OE) [ 949.202328] obdclass(OE) [ 949.202328] ko2iblnd(OE) [ 949.202328] lnet(OE) [ 949.202328] libcfs(OE) [ 949.202329] iTCO_wdt [ 949.202329] iTCO_vendor_support [ 949.202329] sb_edac [ 949.202330] intel_powerclamp [ 949.202330] coretemp [ 949.202330] intel_rapl [ 949.202331] iosf_mbi [ 949.202331] rpcrdma [ 949.202331] ib_iser [ 949.202331] kvm [ 949.202332] irqbypass [ 949.202332] joydev [ 949.202332] pcspkr [ 949.202333] lpc_ich [ 949.202333] i2c_i801 [ 949.202333] ioatdma [ 949.202333] ses [ 949.202334] enclosure [ 949.202334] sg [ 949.202334] ipmi_si [ 949.202334] ipmi_devintf [ 949.202335] ipmi_msghandler [ 949.202335] acpi_power_meter [ 949.202335] dm_round_robin [ 949.202336] acpi_cpufreq [ 949.202336] ib_ipoib [ 949.202336] rdma_ucm [ 949.202336] ib_umad [ 949.202337] sch_fq_codel [ 949.202337] iw_cxgb4 [ 949.202337] rdma_cm [ 949.202338] iw_cm [ 949.202338] ib_cm [ 949.202338] iw_cxgb3 [ 949.202339] zfs(POE) [ 949.202339] binfmt_misc [ 949.202339] zunicode(POE) [ 949.202340] zavl(POE) [ 949.202340] icp(POE) [ 949.202340] zcommon(POE) [ 949.202341] znvpair(POE) [ 949.202341] spl(OE) [ 949.202341] msr_safe(OE) [ 949.202342] ip_tables [ 949.202342] nfsv3 [ 949.202342] nfs_acl [ 949.202342] rpcsec_gss_krb5 [ 949.202343] auth_rpcgss [ 949.202343] nfsv4 [ 949.202343] dns_resolver [ 949.202343] nfs [ 949.202344] lockd [ 949.202344] grace [ 949.202344] fscache [ 949.202345] overlay(T) [ 949.202345] ext4 [ 949.202345] mbcache [ 949.202346] jbd2 [ 949.202346] dm_service_time [ 949.202346] sd_mod [ 949.202347] crc_t10dif [ 949.202347] crct10dif_generic [ 949.202347] mlx5_ib [ 949.202348] ib_uverbs [ 949.202348] ib_core [ 949.202348] be2iscsi [ 949.202348] bnx2i [ 949.202349] cnic [ 949.202349] uio [ 949.202349] cxgb4i [ 949.202349] cxgb4 [ 949.202350] cxgb3i [ 949.202350] cxgb3 [ 949.202350] mdio [ 949.202351] libcxgbi [ 949.202351] libcxgb [ 949.202351] qla4xxx [ 949.202352] iscsi_boot_sysfs [ 949.202352] mgag200 [ 949.202352] i2c_algo_bit [ 949.202352] 8021q [ 949.202353] drm_kms_helper [ 949.202353] garp [ 949.202353] syscopyarea [ 949.202354] mrp [ 949.202354] crct10dif_pclmul [ 949.202354] stp [ 949.202354] sysfillrect [ 949.202355] crct10dif_common [ 949.202355] sysimgblt [ 949.202355] crc32_pclmul [ 949.202356] llc [ 949.202356] fb_sys_fops [ 949.202356] crc32c_intel [ 949.202357] ttm [ 949.202357] ghash_clmulni_intel [ 949.202357] mlx5_core [ 949.202358] drm [ 949.202358] mxm_wmi [ 949.202358] ahci [ 949.202358] ixgbe(OE) [ 949.202359] aesni_intel [ 949.202359] mpt3sas [ 949.202359] mlxfw [ 949.202359] dm_multipath [ 949.202360] lrw [ 949.202360] devlink [ 949.202360] libahci [ 949.202361] gf128mul [ 949.202361] dca [ 949.202361] glue_helper [ 949.202362] ablk_helper [ 949.202362] raid_class [ 949.202362] ptp [ 949.202362] drm_panel_orientation_quirks [ 949.202363] libata [ 949.202363] cryptd [ 949.202363] scsi_transport_sas [ 949.202364] pps_core [ 949.202364] wmi [ 949.202364] sunrpc [ 949.202364] dm_mirror [ 949.202365] dm_region_hash [ 949.202365] dm_log [ 949.202366] dm_mod [ 949.202366] iscsi_tcp [ 949.202366] libiscsi_tcp [ 949.202367] libiscsi [ 949.202367] scsi_transport_iscsi [ 949.202367] fuse [ 949.202368] [ 949.202369] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.202370] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.202371] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 949.202371] RIP: 0010:[] [ 949.202373] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 949.202374] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 949.202374] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 949.202375] RDX: ffff97f2bfd1b8c0 RSI: 0000000000a10000 RDI: ffff97f2a83bcf40 [ 949.202376] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 949.202376] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 949.202377] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 949.202378] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 949.202379] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.202379] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 949.202380] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.202381] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.202381] Call Trace: [ 949.202383] [] queued_spin_lock_slowpath+0xb/0xf [ 949.202385] [] _raw_spin_lock+0x30/0x40 [ 949.202390] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.202396] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.202398] [] ? wake_up_atomic_t+0x30/0x30 [ 949.202404] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.202423] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.202441] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 949.202442] [] ? check_preempt_curr+0x80/0xa0 [ 949.202459] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 949.202477] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 949.202500] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 949.202522] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.202523] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.202542] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.202561] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.202562] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.202563] [] ? sched_feat_set+0xf0/0xf0 [ 949.202581] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.202600] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.202601] [] kthread+0xd1/0xe0 [ 949.202603] [] ? insert_kthread_work+0x40/0x40 [ 949.202604] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.202606] [] ? insert_kthread_work+0x40/0x40 [ 949.202606] Code: [ 949.202606] 13 [ 949.202607] 48 [ 949.202607] c1 [ 949.202607] ea [ 949.202607] 0d [ 949.202608] 48 [ 949.202608] 98 [ 949.202608] 83 [ 949.202608] e2 [ 949.202609] 30 [ 949.202609] 48 [ 949.202609] 81 [ 949.202609] c2 [ 949.202610] c0 [ 949.202610] b8 [ 949.202610] 01 [ 949.202610] 00 [ 949.202611] 48 [ 949.202611] 03 [ 949.202611] 14 [ 949.202612] c5 [ 949.202612] e0 [ 949.202612] 17 [ 949.202612] d5 [ 949.202613] a0 [ 949.202613] 4c [ 949.202613] 89 [ 949.202613] 02 [ 949.202614] 41 [ 949.202614] 8b [ 949.202614] 40 [ 949.202614] 08 [ 949.202615] 85 [ 949.202615] c0 [ 949.202615] 75 [ 949.202615] 0f [ 949.202616] 0f [ 949.202616] 1f [ 949.202616] 44 [ 949.202617] 00 [ 949.202617] 00 [ 949.202617] f3 [ 949.202617] 90 [ 949.202618] <41> [ 949.202618] 8b [ 949.202618] 40 [ 949.202618] 08 [ 949.202619] 85 [ 949.202619] c0 [ 949.202619] 74 [ 949.202620] f6 [ 949.202620] 4d [ 949.202620] 8b [ 949.202620] 08 [ 949.202621] 4d [ 949.202621] 85 [ 949.202621] c9 [ 949.202622] 74 [ 949.202622] 04 [ 949.202622] 41 [ 949.202622] 0f [ 949.202622] 18 [ 949.202623] 09 [ 949.202623] 8b [ 949.202623] [ 949.216324] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ll_mgs_0013:10822] [ 949.216324] Modules linked in: [ 949.216325] osp(OE) [ 949.216326] mdd(OE) [ 949.216326] lod(OE) [ 949.216326] mdt(OE) [ 949.216327] lfsck(OE) [ 949.216327] mgs(OE) [ 949.216328] mgc(OE) [ 949.216328] osd_zfs(OE) [ 949.216328] lquota(OE) [ 949.216328] fid(OE) [ 949.216329] fld(OE) [ 949.216329] ptlrpc(OE) [ 949.216329] obdclass(OE) [ 949.216330] ko2iblnd(OE) [ 949.216330] lnet(OE) [ 949.216330] libcfs(OE) [ 949.216331] iTCO_wdt [ 949.216331] iTCO_vendor_support [ 949.216331] sb_edac [ 949.216331] intel_powerclamp [ 949.216332] coretemp [ 949.216332] intel_rapl [ 949.216332] iosf_mbi [ 949.216332] rpcrdma [ 949.216333] ib_iser [ 949.216333] kvm [ 949.216333] irqbypass [ 949.216334] joydev [ 949.216334] pcspkr [ 949.216334] lpc_ich [ 949.216334] i2c_i801 [ 949.216335] ioatdma [ 949.216335] ses [ 949.216335] enclosure [ 949.216335] sg [ 949.216336] ipmi_si [ 949.216336] ipmi_devintf [ 949.216336] ipmi_msghandler [ 949.216336] acpi_power_meter [ 949.216337] dm_round_robin [ 949.216337] acpi_cpufreq [ 949.216337] ib_ipoib [ 949.216338] rdma_ucm [ 949.216338] ib_umad [ 949.216338] sch_fq_codel [ 949.216338] iw_cxgb4 [ 949.216339] rdma_cm [ 949.216339] iw_cm [ 949.216339] ib_cm [ 949.216339] iw_cxgb3 [ 949.216340] zfs(POE) [ 949.216340] binfmt_misc [ 949.216340] zunicode(POE) [ 949.216341] zavl(POE) [ 949.216341] icp(POE) [ 949.216341] zcommon(POE) [ 949.216342] znvpair(POE) [ 949.216342] spl(OE) [ 949.216342] msr_safe(OE) [ 949.216343] ip_tables [ 949.216343] nfsv3 [ 949.216343] nfs_acl [ 949.216344] rpcsec_gss_krb5 [ 949.216344] auth_rpcgss [ 949.216344] nfsv4 [ 949.216345] dns_resolver [ 949.216345] nfs [ 949.216345] lockd [ 949.216346] grace [ 949.216346] fscache [ 949.216346] overlay(T) [ 949.216347] ext4 [ 949.216347] mbcache [ 949.216347] jbd2 [ 949.216347] dm_service_time [ 949.216348] sd_mod [ 949.216348] crc_t10dif [ 949.216348] crct10dif_generic [ 949.216348] mlx5_ib [ 949.216349] ib_uverbs [ 949.216349] ib_core [ 949.216349] be2iscsi [ 949.216350] bnx2i [ 949.216350] cnic [ 949.216350] uio [ 949.216350] cxgb4i [ 949.216351] cxgb4 [ 949.216351] cxgb3i [ 949.216351] cxgb3 [ 949.216351] mdio [ 949.216352] libcxgbi [ 949.216352] libcxgb [ 949.216352] qla4xxx [ 949.216353] iscsi_boot_sysfs [ 949.216353] mgag200 [ 949.216353] i2c_algo_bit [ 949.216354] 8021q [ 949.216354] drm_kms_helper [ 949.216354] garp [ 949.216355] syscopyarea [ 949.216355] mrp [ 949.216356] crct10dif_pclmul [ 949.216356] stp [ 949.216356] sysfillrect [ 949.216357] crct10dif_common [ 949.216357] sysimgblt [ 949.216357] crc32_pclmul [ 949.216357] llc [ 949.216358] fb_sys_fops [ 949.216358] crc32c_intel [ 949.216358] ttm [ 949.216359] ghash_clmulni_intel [ 949.216359] mlx5_core [ 949.216359] drm [ 949.216360] mxm_wmi [ 949.216360] ahci [ 949.216360] ixgbe(OE) [ 949.216361] aesni_intel [ 949.216361] mpt3sas [ 949.216361] mlxfw [ 949.216361] dm_multipath [ 949.216362] lrw [ 949.216362] devlink [ 949.216362] libahci [ 949.216363] gf128mul [ 949.216363] dca [ 949.216363] glue_helper [ 949.216364] ablk_helper [ 949.216364] raid_class [ 949.216364] ptp [ 949.216365] drm_panel_orientation_quirks [ 949.216365] libata [ 949.216365] cryptd [ 949.216366] scsi_transport_sas [ 949.216366] pps_core [ 949.216366] wmi [ 949.216366] sunrpc [ 949.216367] dm_mirror [ 949.216367] dm_region_hash [ 949.216367] dm_log [ 949.216368] dm_mod [ 949.216368] iscsi_tcp [ 949.216368] libiscsi_tcp [ 949.216368] libiscsi [ 949.216369] scsi_transport_iscsi [ 949.216369] fuse [ 949.216369] [ 949.216371] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.216371] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.216372] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 949.216373] RIP: 0010:[] [ 949.216375] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 949.216375] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 949.216376] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 949.216377] RDX: ffff97f2bfc1b8c0 RSI: 0000000000810000 RDI: ffff97f2a83bcf40 [ 949.216377] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 949.216378] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 949.216378] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 949.216379] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 949.216380] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.216381] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 949.216381] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.216382] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.216382] Call Trace: [ 949.216385] [] queued_spin_lock_slowpath+0xb/0xf [ 949.216386] [] _raw_spin_lock+0x30/0x40 [ 949.216393] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.216400] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.216402] [] ? wake_up_atomic_t+0x30/0x30 [ 949.216408] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.216431] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.216447] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 949.216449] [] ? enqueue_task_fair+0x208/0x6c0 [ 949.216450] [] ? check_preempt_curr+0x80/0xa0 [ 949.216451] [] ? ttwu_do_wakeup+0x19/0x100 [ 949.216475] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 949.216497] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.216498] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.216518] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.216536] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.216537] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.216538] [] ? sched_feat_set+0xf0/0xf0 [ 949.216557] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.216558] [] ? __switch_to+0xce/0x5a0 [ 949.216576] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.216578] [] kthread+0xd1/0xe0 [ 949.216579] [] ? insert_kthread_work+0x40/0x40 [ 949.216581] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.216582] [] ? insert_kthread_work+0x40/0x40 [ 949.216583] Code: [ 949.216583] 13 [ 949.216583] 48 [ 949.216584] c1 [ 949.216584] ea [ 949.216584] 0d [ 949.216584] 48 [ 949.216585] 98 [ 949.216585] 83 [ 949.216585] e2 [ 949.216585] 30 [ 949.216586] 48 [ 949.216586] 81 [ 949.216586] c2 [ 949.216586] c0 [ 949.216587] b8 [ 949.216587] 01 [ 949.216587] 00 [ 949.216587] 48 [ 949.216588] 03 [ 949.216588] 14 [ 949.216588] c5 [ 949.216588] e0 [ 949.216588] 17 [ 949.216589] d5 [ 949.216589] a0 [ 949.216589] 4c [ 949.216589] 89 [ 949.216590] 02 [ 949.216590] 41 [ 949.216590] 8b [ 949.216591] 40 [ 949.216591] 08 [ 949.216591] 85 [ 949.216592] c0 [ 949.216592] 75 [ 949.216592] 0f [ 949.216592] 0f [ 949.216593] 1f [ 949.216593] 44 [ 949.216593] 00 [ 949.216593] 00 [ 949.216593] f3 [ 949.216594] 90 [ 949.216594] <41> [ 949.216594] 8b [ 949.216595] 40 [ 949.216595] 08 [ 949.216595] 85 [ 949.216595] c0 [ 949.216596] 74 [ 949.216596] f6 [ 949.216596] 4d [ 949.216596] 8b [ 949.216597] 08 [ 949.216597] 4d [ 949.216597] 85 [ 949.216597] c9 [ 949.216597] 74 [ 949.216598] 04 [ 949.216598] 41 [ 949.216598] 0f [ 949.216598] 18 [ 949.216599] 09 [ 949.216599] 8b [ 949.216599] [ 949.222324] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ll_mgs_0014:10826] [ 949.222324] Modules linked in: [ 949.222325] osp(OE) [ 949.222326] mdd(OE) [ 949.222326] lod(OE) [ 949.222326] mdt(OE) [ 949.222327] lfsck(OE) [ 949.222327] mgs(OE) [ 949.222327] mgc(OE) [ 949.222328] osd_zfs(OE) [ 949.222328] lquota(OE) [ 949.222328] fid(OE) [ 949.222329] fld(OE) [ 949.222329] ptlrpc(OE) [ 949.222329] obdclass(OE) [ 949.222329] ko2iblnd(OE) [ 949.222330] lnet(OE) [ 949.222330] libcfs(OE) [ 949.222330] iTCO_wdt [ 949.222331] iTCO_vendor_support [ 949.222331] sb_edac [ 949.222331] intel_powerclamp [ 949.222332] coretemp [ 949.222332] intel_rapl [ 949.222332] iosf_mbi [ 949.222332] rpcrdma [ 949.222333] ib_iser [ 949.222333] kvm [ 949.222333] irqbypass [ 949.222333] joydev [ 949.222334] pcspkr [ 949.222334] lpc_ich [ 949.222334] i2c_i801 [ 949.222334] ioatdma [ 949.222335] ses [ 949.222335] enclosure [ 949.222335] sg [ 949.222335] ipmi_si [ 949.222336] ipmi_devintf [ 949.222336] ipmi_msghandler [ 949.222336] acpi_power_meter [ 949.222337] dm_round_robin [ 949.222337] acpi_cpufreq [ 949.222337] ib_ipoib [ 949.222337] rdma_ucm [ 949.222338] ib_umad [ 949.222338] sch_fq_codel [ 949.222338] iw_cxgb4 [ 949.222339] rdma_cm [ 949.222339] iw_cm [ 949.222339] ib_cm [ 949.222339] iw_cxgb3 [ 949.222340] zfs(POE) [ 949.222340] binfmt_misc [ 949.222340] zunicode(POE) [ 949.222341] zavl(POE) [ 949.222341] icp(POE) [ 949.222342] zcommon(POE) [ 949.222342] znvpair(POE) [ 949.222342] spl(OE) [ 949.222342] msr_safe(OE) [ 949.222343] ip_tables [ 949.222343] nfsv3 [ 949.222343] nfs_acl [ 949.222344] rpcsec_gss_krb5 [ 949.222344] auth_rpcgss [ 949.222344] nfsv4 [ 949.222345] dns_resolver [ 949.222345] nfs [ 949.222345] lockd [ 949.222345] grace [ 949.222346] fscache [ 949.222346] overlay(T) [ 949.222347] ext4 [ 949.222347] mbcache [ 949.222347] jbd2 [ 949.222348] dm_service_time [ 949.222348] sd_mod [ 949.222348] crc_t10dif [ 949.222349] crct10dif_generic [ 949.222349] mlx5_ib [ 949.222349] ib_uverbs [ 949.222349] ib_core [ 949.222350] be2iscsi [ 949.222350] bnx2i [ 949.222350] cnic [ 949.222351] uio [ 949.222351] cxgb4i [ 949.222351] cxgb4 [ 949.222351] cxgb3i [ 949.222352] cxgb3 [ 949.222352] mdio [ 949.222352] libcxgbi [ 949.222352] libcxgb [ 949.222353] qla4xxx [ 949.222353] iscsi_boot_sysfs [ 949.222353] mgag200 [ 949.222354] i2c_algo_bit [ 949.222354] 8021q [ 949.222354] drm_kms_helper [ 949.222354] garp [ 949.222355] syscopyarea [ 949.222355] mrp [ 949.222355] crct10dif_pclmul [ 949.222356] stp [ 949.222356] sysfillrect [ 949.222356] crct10dif_common [ 949.222356] sysimgblt [ 949.222357] crc32_pclmul [ 949.222357] llc [ 949.222357] fb_sys_fops [ 949.222357] crc32c_intel [ 949.222358] ttm [ 949.222358] ghash_clmulni_intel [ 949.222358] mlx5_core [ 949.222359] drm [ 949.222359] mxm_wmi [ 949.222359] ahci [ 949.222360] ixgbe(OE) [ 949.222360] aesni_intel [ 949.222360] mpt3sas [ 949.222360] mlxfw [ 949.222361] dm_multipath [ 949.222361] lrw [ 949.222361] devlink [ 949.222361] libahci [ 949.222362] gf128mul [ 949.222362] dca [ 949.222362] glue_helper [ 949.222363] ablk_helper [ 949.222363] raid_class [ 949.222363] ptp [ 949.222364] drm_panel_orientation_quirks [ 949.222364] libata [ 949.222364] cryptd [ 949.222364] scsi_transport_sas [ 949.222365] pps_core [ 949.222365] wmi [ 949.222365] sunrpc [ 949.222366] dm_mirror [ 949.222366] dm_region_hash [ 949.222366] dm_log [ 949.222366] dm_mod [ 949.222367] iscsi_tcp [ 949.222367] libiscsi_tcp [ 949.222367] libiscsi [ 949.222368] scsi_transport_iscsi [ 949.222368] fuse [ 949.222368] [ 949.222370] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.222370] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.222371] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 949.222372] RIP: 0010:[] [ 949.222374] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 949.222375] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 949.222375] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 949.222376] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 949.222377] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 949.222377] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 949.222378] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 949.222379] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 949.222379] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.222380] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 949.222381] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.222381] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.222382] Call Trace: [ 949.222384] [] queued_spin_lock_slowpath+0xb/0xf [ 949.222386] [] _raw_spin_lock+0x30/0x40 [ 949.222391] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 949.222395] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 949.222401] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 949.222403] [] ? wake_up_atomic_t+0x30/0x30 [ 949.222409] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.222427] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.222446] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 949.222447] [] ? check_preempt_curr+0x80/0xa0 [ 949.222464] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 949.222481] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 949.222504] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 949.222525] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.222527] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.222546] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.222565] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.222566] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.222567] [] ? sched_feat_set+0xf0/0xf0 [ 949.222585] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.222587] [] ? __switch_to+0xce/0x5a0 [ 949.222605] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.222606] [] kthread+0xd1/0xe0 [ 949.222608] [] ? insert_kthread_work+0x40/0x40 [ 949.222609] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.222611] [] ? insert_kthread_work+0x40/0x40 [ 949.222611] Code: [ 949.222611] 0d [ 949.222612] 48 [ 949.222612] 98 [ 949.222612] 83 [ 949.222613] e2 [ 949.222613] 30 [ 949.222613] 48 [ 949.222613] 81 [ 949.222614] c2 [ 949.222614] c0 [ 949.222614] b8 [ 949.222614] 01 [ 949.222614] 00 [ 949.222615] 48 [ 949.222615] 03 [ 949.222615] 14 [ 949.222616] c5 [ 949.222616] e0 [ 949.222616] 17 [ 949.222616] d5 [ 949.222616] a0 [ 949.222617] 4c [ 949.222617] 89 [ 949.222617] 02 [ 949.222617] 41 [ 949.222618] 8b [ 949.222618] 40 [ 949.222618] 08 [ 949.222619] 85 [ 949.222619] c0 [ 949.222619] 75 [ 949.222619] 0f [ 949.222620] 0f [ 949.222620] 1f [ 949.222621] 44 [ 949.222621] 00 [ 949.222621] 00 [ 949.222621] f3 [ 949.222622] 90 [ 949.222622] 41 [ 949.222622] 8b [ 949.222622] 40 [ 949.222623] 08 [ 949.222623] <85> [ 949.222623] c0 [ 949.222623] 74 [ 949.222624] f6 [ 949.222624] 4d [ 949.222624] 8b [ 949.222624] 08 [ 949.222625] 4d [ 949.222625] 85 [ 949.222625] c9 [ 949.222625] 74 [ 949.222626] 04 [ 949.222626] 41 [ 949.222626] 0f [ 949.222626] 18 [ 949.222627] 09 [ 949.222627] 8b [ 949.222627] 17 [ 949.222627] 0f [ 949.222628] b7 [ 949.222628] c2 [ 949.222628] [ 949.234326] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 23s! [mdt01_014:17318] [ 949.234326] Modules linked in: [ 949.234327] osp(OE) [ 949.234327] mdd(OE) [ 949.234328] lod(OE) [ 949.234328] mdt(OE) [ 949.234328] lfsck(OE) [ 949.234329] mgs(OE) [ 949.234329] mgc(OE) [ 949.234329] osd_zfs(OE) [ 949.234330] lquota(OE) [ 949.234330] fid(OE) [ 949.234330] fld(OE) [ 949.234330] ptlrpc(OE) [ 949.234331] obdclass(OE) [ 949.234331] ko2iblnd(OE) [ 949.234331] lnet(OE) [ 949.234332] libcfs(OE) [ 949.234332] iTCO_wdt [ 949.234332] iTCO_vendor_support [ 949.234333] sb_edac [ 949.234333] intel_powerclamp [ 949.234333] coretemp [ 949.234333] intel_rapl [ 949.234334] iosf_mbi [ 949.234334] rpcrdma [ 949.234334] ib_iser [ 949.234334] kvm [ 949.234335] irqbypass [ 949.234335] joydev [ 949.234335] pcspkr [ 949.234336] lpc_ich [ 949.234336] i2c_i801 [ 949.234336] ioatdma [ 949.234336] ses [ 949.234337] enclosure [ 949.234337] sg [ 949.234337] ipmi_si [ 949.234337] ipmi_devintf [ 949.234338] ipmi_msghandler [ 949.234338] acpi_power_meter [ 949.234338] dm_round_robin [ 949.234339] acpi_cpufreq [ 949.234339] ib_ipoib [ 949.234339] rdma_ucm [ 949.234339] ib_umad [ 949.234340] sch_fq_codel [ 949.234340] iw_cxgb4 [ 949.234340] rdma_cm [ 949.234340] iw_cm [ 949.234341] ib_cm [ 949.234341] iw_cxgb3 [ 949.234341] zfs(POE) [ 949.234342] binfmt_misc [ 949.234342] zunicode(POE) [ 949.234342] zavl(POE) [ 949.234343] icp(POE) [ 949.234343] zcommon(POE) [ 949.234343] znvpair(POE) [ 949.234344] spl(OE) [ 949.234344] msr_safe(OE) [ 949.234345] ip_tables [ 949.234345] nfsv3 [ 949.234345] nfs_acl [ 949.234345] rpcsec_gss_krb5 [ 949.234346] auth_rpcgss [ 949.234346] nfsv4 [ 949.234347] dns_resolver [ 949.234347] nfs [ 949.234347] lockd [ 949.234347] grace [ 949.234348] fscache [ 949.234348] overlay(T) [ 949.234349] ext4 [ 949.234349] mbcache [ 949.234349] jbd2 [ 949.234349] dm_service_time [ 949.234350] sd_mod [ 949.234350] crc_t10dif [ 949.234350] crct10dif_generic [ 949.234351] mlx5_ib [ 949.234351] ib_uverbs [ 949.234351] ib_core [ 949.234352] be2iscsi [ 949.234352] bnx2i [ 949.234352] cnic [ 949.234353] uio [ 949.234353] cxgb4i [ 949.234353] cxgb4 [ 949.234353] cxgb3i [ 949.234354] cxgb3 [ 949.234354] mdio [ 949.234354] libcxgbi [ 949.234355] libcxgb [ 949.234355] qla4xxx [ 949.234355] iscsi_boot_sysfs [ 949.234356] mgag200 [ 949.234356] i2c_algo_bit [ 949.234356] 8021q [ 949.234356] drm_kms_helper [ 949.234357] garp [ 949.234357] syscopyarea [ 949.234357] mrp [ 949.234358] crct10dif_pclmul [ 949.234358] stp [ 949.234358] sysfillrect [ 949.234359] crct10dif_common [ 949.234359] sysimgblt [ 949.234359] crc32_pclmul [ 949.234359] llc [ 949.234360] fb_sys_fops [ 949.234360] crc32c_intel [ 949.234360] ttm [ 949.234361] ghash_clmulni_intel [ 949.234361] mlx5_core [ 949.234361] drm [ 949.234361] mxm_wmi [ 949.234362] ahci [ 949.234362] ixgbe(OE) [ 949.234362] aesni_intel [ 949.234363] mpt3sas [ 949.234363] mlxfw [ 949.234363] dm_multipath [ 949.234363] lrw [ 949.234364] devlink [ 949.234364] libahci [ 949.234364] gf128mul [ 949.234365] dca [ 949.234365] glue_helper [ 949.234365] ablk_helper [ 949.234365] raid_class [ 949.234366] ptp [ 949.234366] drm_panel_orientation_quirks [ 949.234366] libata [ 949.234366] cryptd [ 949.234367] scsi_transport_sas [ 949.234367] pps_core [ 949.234367] wmi [ 949.234368] sunrpc [ 949.234368] dm_mirror [ 949.234368] dm_region_hash [ 949.234368] dm_log [ 949.234369] dm_mod [ 949.234369] iscsi_tcp [ 949.234369] libiscsi_tcp [ 949.234369] libiscsi [ 949.234370] scsi_transport_iscsi [ 949.234370] fuse [ 949.234370] [ 949.234372] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.234372] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.234373] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 949.234374] RIP: 0010:[] [ 949.234376] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 949.234377] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 949.234378] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 949.234378] RDX: ffff97f2bfa1b8c0 RSI: 0000000000010000 RDI: ffff97f2a83bcf40 [ 949.234379] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 949.234380] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 949.234380] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 949.234382] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 949.234382] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.234383] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 949.234384] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.234384] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.234385] Call Trace: [ 949.234387] [] queued_spin_lock_slowpath+0xb/0xf [ 949.234388] [] _raw_spin_lock+0x30/0x40 [ 949.234394] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.234400] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.234402] [] ? wake_up_atomic_t+0x30/0x30 [ 949.234407] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.234425] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.234444] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 949.234462] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 949.234480] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 949.234497] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 949.234520] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 949.234541] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.234542] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.234562] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.234580] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.234581] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.234582] [] ? sched_feat_set+0xf0/0xf0 [ 949.234601] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.234619] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.234620] [] kthread+0xd1/0xe0 [ 949.234622] [] ? insert_kthread_work+0x40/0x40 [ 949.234623] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.234624] [] ? insert_kthread_work+0x40/0x40 [ 949.234625] Code: [ 949.234625] c1 [ 949.234626] e8 [ 949.234626] 13 [ 949.234626] 48 [ 949.234626] c1 [ 949.234627] ea [ 949.234627] 0d [ 949.234627] 48 [ 949.234627] 98 [ 949.234628] 83 [ 949.234628] e2 [ 949.234628] 30 [ 949.234629] 48 [ 949.234629] 81 [ 949.234629] c2 [ 949.234629] c0 [ 949.234630] b8 [ 949.234630] 01 [ 949.234630] 00 [ 949.234631] 48 [ 949.234631] 03 [ 949.234631] 14 [ 949.234631] c5 [ 949.234632] e0 [ 949.234632] 17 [ 949.234632] d5 [ 949.234632] a0 [ 949.234633] 4c [ 949.234633] 89 [ 949.234633] 02 [ 949.234634] 41 [ 949.234634] 8b [ 949.234634] 40 [ 949.234634] 08 [ 949.234635] 85 [ 949.234635] c0 [ 949.234635] 75 [ 949.234635] 0f [ 949.234636] 0f [ 949.234636] 1f [ 949.234636] 44 [ 949.234636] 00 [ 949.234637] 00 [ 949.234637] [ 949.234637] 90 [ 949.234638] 41 [ 949.234638] 8b [ 949.234638] 40 [ 949.234638] 08 [ 949.234638] 85 [ 949.234639] c0 [ 949.234639] 74 [ 949.234639] f6 [ 949.234639] 4d [ 949.234640] 8b [ 949.234640] 08 [ 949.234640] 4d [ 949.234640] 85 [ 949.234641] c9 [ 949.234641] 74 [ 949.234641] 04 [ 949.234641] 41 [ 949.234642] 0f [ 949.234642] 18 [ 949.234642] [ 949.239326] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 23s! [mdt01_017:17555] [ 949.239326] Modules linked in: [ 949.239326] osp(OE) [ 949.239327] mdd(OE) [ 949.239327] lod(OE) [ 949.239328] mdt(OE) [ 949.239328] lfsck(OE) [ 949.239329] mgs(OE) [ 949.239329] mgc(OE) [ 949.239329] osd_zfs(OE) [ 949.239330] lquota(OE) [ 949.239330] fid(OE) [ 949.239331] fld(OE) [ 949.239331] ptlrpc(OE) [ 949.239332] obdclass(OE) [ 949.239332] ko2iblnd(OE) [ 949.239332] lnet(OE) [ 949.239333] libcfs(OE) [ 949.239333] iTCO_wdt [ 949.239334] iTCO_vendor_support [ 949.239334] sb_edac [ 949.239334] intel_powerclamp [ 949.239335] coretemp [ 949.239335] intel_rapl [ 949.239336] iosf_mbi [ 949.239336] rpcrdma [ 949.239336] ib_iser [ 949.239337] kvm [ 949.239337] irqbypass [ 949.239337] joydev [ 949.239338] pcspkr [ 949.239338] lpc_ich [ 949.239339] i2c_i801 [ 949.239339] ioatdma [ 949.239339] ses [ 949.239340] enclosure [ 949.239340] sg [ 949.239340] ipmi_si [ 949.239341] ipmi_devintf [ 949.239341] ipmi_msghandler [ 949.239342] acpi_power_meter [ 949.239342] dm_round_robin [ 949.239342] acpi_cpufreq [ 949.239343] ib_ipoib [ 949.239343] rdma_ucm [ 949.239343] ib_umad [ 949.239344] sch_fq_codel [ 949.239344] iw_cxgb4 [ 949.239345] rdma_cm [ 949.239345] iw_cm [ 949.239345] ib_cm [ 949.239346] iw_cxgb3 [ 949.239346] zfs(POE) [ 949.239347] binfmt_misc [ 949.239347] zunicode(POE) [ 949.239347] zavl(POE) [ 949.239348] icp(POE) [ 949.239348] zcommon(POE) [ 949.239349] znvpair(POE) [ 949.239349] spl(OE) [ 949.239350] msr_safe(OE) [ 949.239350] ip_tables [ 949.239350] nfsv3 [ 949.239351] nfs_acl [ 949.239351] rpcsec_gss_krb5 [ 949.239352] auth_rpcgss [ 949.239352] nfsv4 [ 949.239353] dns_resolver [ 949.239353] nfs [ 949.239353] lockd [ 949.239354] grace [ 949.239354] fscache [ 949.239355] overlay(T) [ 949.239355] ext4 [ 949.239356] mbcache [ 949.239356] jbd2 [ 949.239356] dm_service_time [ 949.239357] sd_mod [ 949.239357] crc_t10dif [ 949.239358] crct10dif_generic [ 949.239358] mlx5_ib [ 949.239359] ib_uverbs [ 949.239359] ib_core [ 949.239359] be2iscsi [ 949.239360] bnx2i [ 949.239361] cnic [ 949.239361] uio [ 949.239361] cxgb4i [ 949.239362] cxgb4 [ 949.239362] cxgb3i [ 949.239363] cxgb3 [ 949.239363] mdio [ 949.239363] libcxgbi [ 949.239364] libcxgb [ 949.239364] qla4xxx [ 949.239365] iscsi_boot_sysfs [ 949.239365] mgag200 [ 949.239365] i2c_algo_bit [ 949.239366] 8021q [ 949.239366] drm_kms_helper [ 949.239366] garp [ 949.239367] syscopyarea [ 949.239367] mrp [ 949.239368] crct10dif_pclmul [ 949.239368] stp [ 949.239368] sysfillrect [ 949.239369] crct10dif_common [ 949.239369] sysimgblt [ 949.239370] crc32_pclmul [ 949.239370] llc [ 949.239370] fb_sys_fops [ 949.239371] crc32c_intel [ 949.239371] ttm [ 949.239372] ghash_clmulni_intel [ 949.239372] mlx5_core [ 949.239372] drm [ 949.239373] mxm_wmi [ 949.239373] ahci [ 949.239374] ixgbe(OE) [ 949.239374] aesni_intel [ 949.239375] mpt3sas [ 949.239375] mlxfw [ 949.239375] dm_multipath [ 949.239376] lrw [ 949.239376] devlink [ 949.239377] libahci [ 949.239377] gf128mul [ 949.239377] dca [ 949.239378] glue_helper [ 949.239378] ablk_helper [ 949.239379] raid_class [ 949.239379] ptp [ 949.239379] drm_panel_orientation_quirks [ 949.239380] libata [ 949.239380] cryptd [ 949.239381] scsi_transport_sas [ 949.239381] pps_core [ 949.239381] wmi [ 949.239382] sunrpc [ 949.239382] dm_mirror [ 949.239382] dm_region_hash [ 949.239383] dm_log [ 949.239383] dm_mod [ 949.239384] iscsi_tcp [ 949.239384] libiscsi_tcp [ 949.239384] libiscsi [ 949.239385] scsi_transport_iscsi [ 949.239385] fuse [ 949.239386] [ 949.239388] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.239388] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.239389] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 949.239390] RIP: 0010:[] [ 949.239392] [] native_queued_spin_lock_slowpath+0x156/0x200 [ 949.239393] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000202 [ 949.239394] RAX: 0000000000000001 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 949.239395] RDX: 0000000000710001 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 949.239396] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: ffff9832bf59b8c0 [ 949.239397] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 949.239398] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 949.239399] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 949.239400] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.239401] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 949.239402] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.239403] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.239403] Call Trace: [ 949.239406] [] queued_spin_lock_slowpath+0xb/0xf [ 949.239407] [] _raw_spin_lock+0x30/0x40 [ 949.239414] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.239422] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.239424] [] ? wake_up_atomic_t+0x30/0x30 [ 949.239431] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.239455] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.239479] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 949.239481] [] ? check_preempt_curr+0x80/0xa0 [ 949.239504] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 949.239528] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 949.239558] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 949.239586] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.239588] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.239613] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.239638] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.239640] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.239701] iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 949.239703] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.239704] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.239705] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 949.239713] RIP: 0010:[] [] lnet_discover_peer_locked+0x1f1/0x450 [lnet] [ 949.239713] RSP: 0018:ffff98321b877aa8 EFLAGS: 00000206 [ 949.239714] RAX: ffff98321b877fd8 RBX: ffffffffffffff10 RCX: 0000000000110000 [ 949.239714] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff97f26a12604c [ 949.239715] RBP: ffff98321b877b28 R08: ffff97f2bfa9b8c0 R09: ffff9832bf55b8c0 [ 949.239716] R10: 0000000000000001 R11: 000000000000242e R12: 0000000000710000 [ 949.239716] R13: ffff9832bf39b8c0 R14: 0000000000110000 R15: 0000000000000000 [ 949.239717] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 949.239718] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.239718] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 949.239719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.239719] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.239720] Call Trace: [ 949.239722] [] ? wake_up_atomic_t+0x30/0x30 [ 949.239728] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.239747] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.239763] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 949.239781] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 949.239783] [] ? enqueue_task_fair+0x208/0x6c0 [ 949.239784] [] ? sched_clock_cpu+0xa5/0xe0 [ 949.239785] [] ? check_preempt_curr+0x80/0xa0 [ 949.239786] [] ? ttwu_do_wakeup+0x19/0x100 [ 949.239810] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 949.239831] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.239832] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.239852] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.239872] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.239874] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.239875] [] ? sched_feat_set+0xf0/0xf0 [ 949.239893] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.239895] [] ? __switch_to+0xce/0x5a0 [ 949.239913] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.239915] [] kthread+0xd1/0xe0 [ 949.239916] [] ? insert_kthread_work+0x40/0x40 [ 949.239918] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.239919] [] ? insert_kthread_work+0x40/0x40 [ 949.239933] Code: ff ff e8 63 f2 97 ff f0 41 ff 0c 24 0f 94 c0 84 c0 74 08 48 89 df e8 ef 7a ff ff 49 8b 47 70 48 8b 58 20 4c 8d 63 48 f0 ff 43 48 <4c> 8d ab a8 00 00 00 48 8d 75 a8 ba 01 00 00 00 4c 89 ef e8 77 [ 949.241326] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [mdt01_018:17592] [ 949.241355] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 949.241376] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 949.241377] CPU: 30 PID: 17592 Comm: mdt01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 949.241378] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 949.241379] task: ffff983230579080 ti: ffff9832469b0000 task.ti: ffff9832469b0000 [ 949.241382] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 949.241383] RSP: 0018:ffff9832469b3a50 EFLAGS: 00000246 [ 949.241384] RAX: 0000000000000000 RBX: ffff9832bf580000 RCX: 0000000000f10000 [ 949.241385] RDX: ffff9832bf55b8c0 RSI: 0000000000e90000 RDI: ffff97f2a83bcf40 [ 949.241385] RBP: ffff9832469b3a50 R08: ffff9832bf59b8c0 R09: 0000000000000000 [ 949.241386] R10: 0000000000000002 R11: ffff983211336600 R12: ffff9832469b3a18 [ 949.241387] R13: ffffffffa00ec9b1 R14: ffff9832469b39f8 R15: ffff9832469b3a28 [ 949.241388] FS: 0000000000000000(0000) GS:ffff9832bf580000(0000) knlGS:0000000000000000 [ 949.241389] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 949.241390] CR2: 00007ffff7fc5010 CR3: 0000003f74514000 CR4: 00000000003607e0 [ 949.241391] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 949.241392] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 949.241392] Call Trace: [ 949.241394] [] queued_spin_lock_slowpath+0xb/0xf [ 949.241396] [] _raw_spin_lock+0x30/0x40 [ 949.241402] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 949.241410] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 949.241412] [] ? wake_up_atomic_t+0x30/0x30 [ 949.241420] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 949.241445] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 949.241469] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 949.241471] [] ? check_preempt_curr+0x80/0xa0 [ 949.241472] [] ? ttwu_do_wakeup+0x19/0x100 [ 949.241503] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 949.241531] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 949.241533] [] ? __getnstimeofday64+0x3f/0xd0 [ 949.241560] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 949.241584] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 949.241586] [] ? __wake_up_common_lock+0x91/0xc0 [ 949.241587] [] ? sched_feat_set+0xf0/0xf0 [ 949.241611] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 949.241635] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 949.241637] [] kthread+0xd1/0xe0 [ 949.241639] [] ? insert_kthread_work+0x40/0x40 [ 949.241641] [] ret_from_fork_nospec_begin+0x21/0x21 [ 949.241643] [] ? insert_kthread_work+0x40/0x40 [ 949.241662] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 953.476640] [] ? sched_feat_set+0xf0/0xf0 [ 953.483466] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 953.492496] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 953.502766] [] kthread+0xd1/0xe0 [ 953.508697] [] ? insert_kthread_work+0x40/0x40 [ 953.517528] [] ret_from_fork_nospec_begin+0x21/0x21 [ 953.525298] [] ? insert_kthread_work+0x40/0x40 [ 953.534121] Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 [ 953.564174] Lustre: MGS: Received new MDS connection from 192.168.129.155@o2ib27, keep former export from same NID [ 953.576293] Lustre: Skipped 527 previous similar messages [ 954.578951] LNet: Service thread pid 17594 was inactive for 200.36s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [ 954.598362] LNet: Skipped 1 previous similar message [ 954.604389] Pid: 17594, comm: mdt00_013 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 SMP Tue Jan 25 12:06:24 PST 2022 [ 954.616033] Call Trace: [ 954.619242] [] 0xffffffffffffffff [ 954.625293] LustreError: dumping log to /tmp/lustre-log.1644342610.17594 [ 961.207704] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [mdt00_031:18467] [ 961.211704] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [mdt00_030:16183] [ 961.211731] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 961.211752] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 961.211754] CPU: 17 PID: 16183 Comm: mdt00_030 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 961.211755] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 961.211756] task: ffff97f250d53180 ti: ffff97f250d78000 task.ti: ffff97f250d78000 [ 961.211759] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 961.211760] RSP: 0018:ffff97f250d7bab0 EFLAGS: 00000246 [ 961.211760] RAX: 0000000000000000 RBX: ffff97f2bfc40000 RCX: 0000000000890000 [ 961.211761] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 961.211762] RBP: ffff97f250d7bab0 R08: ffff97f2bfc5b8c0 R09: 0000000000000000 [ 961.211762] R10: 0000000000000001 R11: 0000000000000400 R12: ffff97f250d7ba78 [ 961.211763] R13: ffffffffa00ec9b1 R14: ffff97f250d7ba58 R15: ffff97f250d7ba88 [ 961.211764] FS: 0000000000000000(0000) GS:ffff97f2bfc40000(0000) knlGS:0000000000000000 [ 961.211764] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 961.211765] CR2: 00007ffff75f34b1 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 961.211766] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 961.211766] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 961.211767] Call Trace: [ 961.211770] [] queued_spin_lock_slowpath+0xb/0xf [ 961.211772] [] _raw_spin_lock+0x30/0x40 [ 961.211784] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 961.211789] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 961.211801] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 961.211803] [] ? wake_up_atomic_t+0x30/0x30 [ 961.211809] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 961.211850] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 961.211868] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 961.211870] [] ? check_preempt_curr+0x80/0xa0 [ 961.211887] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 961.211904] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 961.211932] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 961.211954] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 961.211955] [] ? __getnstimeofday64+0x3f/0xd0 [ 961.211975] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 961.211993] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 961.211994] [] ? __wake_up_common_lock+0x91/0xc0 [ 961.211995] [] ? sched_feat_set+0xf0/0xf0 [ 961.212014] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 961.212016] [] ? __switch_to+0xce/0x5a0 [ 961.212034] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 961.212035] [] kthread+0xd1/0xe0 [ 961.212037] [] ? insert_kthread_work+0x40/0x40 [ 961.212038] [] ret_from_fork_nospec_begin+0x21/0x21 [ 961.212039] [] ? insert_kthread_work+0x40/0x40 [ 961.212053] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 961.725524] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 961.805371] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 961.866454] CPU: 16 PID: 18467 Comm: mdt00_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 961.880815] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 961.893141] task: ffff97f249f7a100 ti: ffff97f249078000 task.ti: ffff97f249078000 [ 961.901976] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 961.913539] RSP: 0018:ffff97f24907bab0 EFLAGS: 00000246 [ 961.919950] RAX: 0000000000000000 RBX: ffff97f2bfc00000 RCX: 0000000000810000 [ 961.928397] RDX: ffff9832bf59b8c0 RSI: 0000000000f10000 RDI: ffff97f2a83bcf40 [ 961.936844] RBP: ffff97f24907bab0 R08: ffff97f2bfc1b8c0 R09: 0000000000000000 [ 961.945290] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f24907ba78 [ 961.953738] R13: ffffffffa00ec9b1 R14: ffff97f24907ba58 R15: ffff97f24907ba88 [ 961.962186] FS: 0000000000000000(0000) GS:ffff97f2bfc00000(0000) knlGS:0000000000000000 [ 961.971700] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 961.978596] CR2: 00007ffff7ad2d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 961.987044] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 961.995490] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 962.003937] Call Trace: [ 962.007152] [] queued_spin_lock_slowpath+0xb/0xf [ 962.014629] [] _raw_spin_lock+0x30/0x40 [ 962.021241] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 962.028916] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 962.036888] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 962.045432] [] ? wake_up_atomic_t+0x30/0x30 [ 962.052431] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 962.059837] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 962.068114] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 962.076101] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 962.084766] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 962.092654] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 962.099968] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 962.108154] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 962.117087] [] ? __getnstimeofday64+0x3f/0xd0 [ 962.124300] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 962.133353] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 962.141415] [] ? __wake_up_common_lock+0x91/0xc0 [ 962.148892] [] ? sched_feat_set+0xf0/0xf0 [ 962.155715] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 962.163195] [] ? __switch_to+0xce/0x5a0 [ 962.169824] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 962.178563] [] kthread+0xd1/0xe0 [ 962.184492] [] ? insert_kthread_work+0x40/0x40 [ 962.191776] [] ret_from_fork_nospec_begin+0x21/0x21 [ 962.196850] INFO: task systemd:1 blocked for more than 120 seconds. [ 962.196851] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 962.196853] systemd D ffff97b4dec585e0 0 1 0 0x00000080 [ 962.196854] Call Trace: [ 962.196860] [] schedule+0x29/0x70 [ 962.196862] [] schedule_timeout+0x271/0x320 [ 962.196865] [] ? sched_clock+0x9/0x10 [ 962.196868] [] ? try_to_wake_up+0x19c/0x3c0 [ 962.196869] [] wait_for_completion+0x12d/0x170 [ 962.196871] [] ? wake_up_state+0x20/0x20 [ 962.196875] [] flush_work+0x12d/0x1f0 [ 962.196877] [] ? move_linked_works+0x90/0x90 [ 962.196882] [] lru_add_drain_all+0x152/0x1a0 [ 962.196886] [] mem_cgroup_reparent_charges+0x38/0x420 [ 962.196891] [] ? d_tmpfile+0xd0/0xd0 [ 962.196892] [] ? d_walk+0x10b/0x3f0 [ 962.196893] [] mem_cgroup_css_offline+0xbc/0x180 [ 962.196897] [] cgroup_destroy_locked+0xea/0x3a0 [ 962.196899] [] cgroup_rmdir+0x22/0x40 [ 962.196900] [] vfs_rmdir+0xdc/0x160 [ 962.196901] [] do_rmdir+0x1f1/0x220 [ 962.196903] [] ? task_work_run+0xd3/0xf0 [ 962.196905] [] SyS_rmdir+0x16/0x20 [ 962.196906] [] system_call_fastpath+0x25/0x2a [ 962.196926] INFO: task kworker/24:1:228 blocked for more than 120 seconds. [ 962.196926] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 962.196928] kworker/24:1 D ffff9832b51d8000 0 228 2 0x00000080 [ 962.196930] Workqueue: events cgroup_release_agent [ 962.196931] Call Trace: [ 962.196932] [] schedule_preempt_disabled+0x39/0x90 [ 962.196934] [] __mutex_lock_slowpath+0x10f/0x250 [ 962.196936] [] mutex_lock+0x32/0x42 [ 962.196937] [] cgroup_release_agent+0xeb/0x1d0 [ 962.196939] [] process_one_work+0x18f/0x4a0 [ 962.196941] [] worker_thread+0x126/0x3e0 [ 962.196942] [] ? rescuer_thread+0x430/0x430 [ 962.196944] [] kthread+0xd1/0xe0 [ 962.196945] [] ? insert_kthread_work+0x40/0x40 [ 962.196946] [] ret_from_fork_nospec_begin+0x21/0x21 [ 962.196948] [] ? insert_kthread_work+0x40/0x40 [ 962.196986] INFO: task kiblnd_connd:30503 blocked for more than 120 seconds. [ 962.196987] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 962.196988] kiblnd_connd D ffff98324e563760 0 30503 2 0x00000080 [ 962.196988] Call Trace: [ 962.196990] [] schedule+0x29/0x70 [ 962.196992] [] schedule_timeout+0x271/0x320 [ 962.196995] [] ? __slab_free+0x1b0/0x290 [ 962.196996] [] wait_for_completion+0x12d/0x170 [ 962.196998] [] ? wake_up_state+0x20/0x20 [ 962.197005] [] cm_destroy_id+0x18a/0x510 [ib_cm] [ 962.197008] [] ib_destroy_cm_id+0x10/0x20 [ib_cm] [ 962.197011] [] rdma_destroy_id+0xa0/0x270 [rdma_cm] [ 962.197017] [] kiblnd_destroy_conn+0x22c/0x680 [ko2iblnd] [ 962.197022] [] kiblnd_connd+0x6a7/0xa00 [ko2iblnd] [ 962.197023] [] ? wake_up_state+0x20/0x20 [ 962.197027] [] ? kiblnd_cm_callback+0x2390/0x2390 [ko2iblnd] [ 962.197028] [] kthread+0xd1/0xe0 [ 962.197030] [] ? insert_kthread_work+0x40/0x40 [ 962.197031] [] ret_from_fork_nospec_begin+0x21/0x21 [ 962.197032] [] ? insert_kthread_work+0x40/0x40 [ 962.197070] INFO: task mdt00_027:16160 blocked for more than 120 seconds. [ 962.197070] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 962.197072] mdt00_027 D ffff97f2513b1660 0 16160 2 0x00000080 [ 962.197072] Call Trace: [ 962.197074] [] schedule+0x29/0x70 [ 962.197075] [] rwsem_down_write_failed+0x1b5/0x400 [ 962.197079] [] call_rwsem_down_write_failed+0x17/0x30 [ 962.197081] [] down_write+0x40/0x50 [ 962.197100] [] lod_qos_statfs_update+0x97/0x2b0 [lod] [ 962.197105] [] lod_qos_prep_create+0x168/0x18b0 [lod] [ 962.197111] [] ? spl_kmem_free+0x35/0x40 [spl] [ 962.197120] [] ? qsd_op_begin+0xb1/0x4e0 [lquota] [ 962.197126] [] lod_prepare_create+0x215/0x2e0 [lod] [ 962.197131] [] lod_declare_striped_create+0x1ee/0x980 [lod] [ 962.197136] [] ? lod_sub_declare_create+0xdf/0x210 [lod] [ 962.197141] [] lod_declare_create+0x204/0x590 [lod] [ 962.197151] [] mdd_declare_create_object_internal+0xea/0x360 [mdd] [ 962.197156] [] mdd_declare_create+0x4c/0xcb0 [mdd] [ 962.197160] [] mdd_create+0x867/0x14a0 [mdd] [ 962.197179] [] mdt_reint_open+0x2598/0x3990 [mdt] [ 962.197204] [] ? check_unlink_entry+0x19/0xd0 [obdclass] [ 962.197215] [] ? upcall_cache_get_entry+0x238/0x920 [obdclass] [ 962.197222] [] ? ucred_set_audit_enabled.isra.15+0x22/0x60 [mdt] [ 962.197229] [] mdt_reint_rec+0x83/0x210 [mdt] [ 962.197235] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [ 962.197242] [] ? mdt_intent_fixup_resent+0x36/0x220 [mdt] [ 962.197248] [] mdt_intent_open+0x82/0x3a0 [mdt] [ 962.197253] [] mdt_intent_opc+0x1ba/0xb50 [mdt] [ 962.197287] [] ? lustre_swab_ldlm_policy_data+0x30/0x30 [ptlrpc] [ 962.197294] [] ? mdt_intent_fixup_resent+0x220/0x220 [mdt] [ 962.197299] [] mdt_intent_policy+0x1a4/0x360 [mdt] [ 962.197314] [] ldlm_lock_enqueue+0x376/0x9b0 [ptlrpc] [ 962.197321] [] ? cfs_hash_bd_add_locked+0x67/0x90 [libcfs] [ 962.197325] [] ? cfs_hash_add+0xbe/0x1a0 [libcfs] [ 962.197342] [] ldlm_handle_enqueue0+0xa86/0x1620 [ptlrpc] [ 962.197361] [] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc] [ 962.197387] [] tgt_enqueue+0x62/0x210 [ptlrpc] [ 962.197413] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [ 962.197439] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 962.197444] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [ 962.197470] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 962.197495] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 962.197496] [] ? __wake_up_common_lock+0x91/0xc0 [ 962.197497] [] ? sched_feat_set+0xf0/0xf0 [ 962.197523] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 962.197526] [] ? __switch_to+0xce/0x5a0 [ 962.197551] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 962.197553] [] kthread+0xd1/0xe0 [ 962.197554] [] ? insert_kthread_work+0x40/0x40 [ 962.197556] [] ret_from_fork_nospec_begin+0x21/0x21 [ 962.197557] [] ? insert_kthread_work+0x40/0x40 [ 963.017654] [] ? insert_kthread_work+0x40/0x40 [ 963.024938] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 965.066147] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [mdt00_008:17320] [ 965.080149] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_mgs_0009:8655] [ 965.075177] Modules linked in: osp(OE) mdd(OE) [ 965.080150] Modules linked in: [ 965.080151] osp(OE) [ 965.080151] mdd(OE) [ 965.080152] lod(OE) [ 965.080152] mdt(OE) [ 965.080152] lfsck(OE) [ 965.080153] mgs(OE) [ 965.080153] mgc(OE) [ 965.080153] osd_zfs(OE) [ 965.080153] lquota(OE) [ 965.080154] fid(OE) [ 965.080154] fld(OE) [ 965.080154] ptlrpc(OE) [ 965.080155] obdclass(OE) [ 965.080155] ko2iblnd(OE) [ 965.080155] lnet(OE) [ 965.080156] libcfs(OE) [ 965.080156] iTCO_wdt [ 965.080156] iTCO_vendor_support [ 965.080157] sb_edac [ 965.080157] intel_powerclamp [ 965.080157] coretemp [ 965.080158] intel_rapl [ 965.080158] iosf_mbi [ 965.080158] rpcrdma [ 965.080158] ib_iser [ 965.080159] kvm [ 965.080159] irqbypass [ 965.080159] joydev [ 965.080159] pcspkr [ 965.080160] lpc_ich [ 965.080160] i2c_i801 [ 965.080160] ioatdma [ 965.080161] ses [ 965.080161] enclosure [ 965.080161] sg [ 965.080161] ipmi_si [ 965.080162] ipmi_devintf [ 965.080162] ipmi_msghandler [ 965.080162] acpi_power_meter [ 965.080163] dm_round_robin [ 965.080163] acpi_cpufreq [ 965.080163] ib_ipoib [ 965.080163] rdma_ucm [ 965.080164] ib_umad [ 965.080164] sch_fq_codel [ 965.080164] iw_cxgb4 [ 965.080165] rdma_cm [ 965.080165] iw_cm [ 965.080166] ib_cm [ 965.080166] iw_cxgb3 [ 965.080166] zfs(POE) [ 965.080166] binfmt_misc [ 965.080167] zunicode(POE) [ 965.080167] zavl(POE) [ 965.080168] icp(POE) [ 965.080168] zcommon(POE) [ 965.080168] znvpair(POE) [ 965.080169] spl(OE) [ 965.080169] msr_safe(OE) [ 965.080170] ip_tables [ 965.080170] nfsv3 [ 965.080170] nfs_acl [ 965.080171] rpcsec_gss_krb5 [ 965.080171] auth_rpcgss [ 965.080171] nfsv4 [ 965.080172] dns_resolver [ 965.080172] nfs [ 965.080172] lockd [ 965.080172] grace [ 965.080173] fscache [ 965.080173] overlay(T) [ 965.080173] ext4 [ 965.080174] mbcache [ 965.080174] jbd2 [ 965.080174] dm_service_time [ 965.080175] sd_mod [ 965.080175] crc_t10dif [ 965.080175] crct10dif_generic [ 965.080176] mlx5_ib [ 965.080176] ib_uverbs [ 965.080176] ib_core [ 965.080177] be2iscsi [ 965.080177] bnx2i [ 965.080177] cnic [ 965.080178] uio [ 965.080178] cxgb4i [ 965.080178] cxgb4 [ 965.080179] cxgb3i [ 965.080179] cxgb3 [ 965.080179] mdio [ 965.080180] libcxgbi [ 965.080180] libcxgb [ 965.080180] qla4xxx [ 965.080180] iscsi_boot_sysfs [ 965.080181] mgag200 [ 965.080181] i2c_algo_bit [ 965.080181] 8021q [ 965.080182] drm_kms_helper [ 965.080182] garp [ 965.080182] syscopyarea [ 965.080183] mrp [ 965.080183] crct10dif_pclmul [ 965.080183] stp [ 965.080184] sysfillrect [ 965.080184] crct10dif_common [ 965.080185] sysimgblt [ 965.080185] crc32_pclmul [ 965.080185] llc [ 965.080185] fb_sys_fops [ 965.080186] crc32c_intel [ 965.080186] ttm [ 965.080186] ghash_clmulni_intel [ 965.080187] mlx5_core [ 965.080187] drm [ 965.080187] mxm_wmi [ 965.080187] ahci [ 965.080188] ixgbe(OE) [ 965.080188] aesni_intel [ 965.080188] mpt3sas [ 965.080189] mlxfw [ 965.080189] dm_multipath [ 965.080189] lrw [ 965.080190] devlink [ 965.080190] libahci [ 965.080190] gf128mul [ 965.080191] dca [ 965.080191] glue_helper [ 965.080192] ablk_helper [ 965.080192] raid_class [ 965.080192] ptp [ 965.080193] drm_panel_orientation_quirks [ 965.080193] libata [ 965.080193] cryptd [ 965.080193] scsi_transport_sas [ 965.080194] pps_core [ 965.080194] wmi [ 965.080194] sunrpc [ 965.080195] dm_mirror [ 965.080195] dm_region_hash [ 965.080195] dm_log [ 965.080195] dm_mod [ 965.080196] iscsi_tcp [ 965.080196] libiscsi_tcp [ 965.080196] libiscsi [ 965.080197] scsi_transport_iscsi [ 965.080197] fuse [ 965.080197] [ 965.080199] CPU: 3 PID: 8655 Comm: ll_mgs_0009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 965.080200] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 965.080200] task: ffff983229036300 ti: ffff98322904c000 task.ti: ffff98322904c000 [ 965.080201] RIP: 0010:[] [ 965.080203] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 965.080204] RSP: 0018:ffff98322904fa50 EFLAGS: 00000246 [ 965.080205] RAX: 0000000000000000 RBX: ffff97f2bfac0000 RCX: 0000000000190000 [ 965.080205] RDX: ffff9832bf59b8c0 RSI: 0000000000f10000 RDI: ffff97f2a83bcf40 [ 965.080206] RBP: ffff98322904fa50 R08: ffff97f2bfadb8c0 R09: 0000000000000000 [ 965.080207] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98322904fa18 [ 965.080207] R13: ffffffffa00ec9b1 R14: ffff98322904f9f8 R15: ffff98322904fa28 [ 965.080208] FS: 0000000000000000(0000) GS:ffff97f2bfac0000(0000) knlGS:0000000000000000 [ 965.080209] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 965.080209] CR2: 00007ffff579f624 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 965.080210] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 965.080211] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 965.080211] Call Trace: [ 965.080213] [] queued_spin_lock_slowpath+0xb/0xf [ 965.080215] [] _raw_spin_lock+0x30/0x40 [ 965.080223] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 965.080227] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 965.080236] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 965.080238] [] ? wake_up_atomic_t+0x30/0x30 [ 965.080245] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 965.080276] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 965.080298] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 965.080305] [] ? lnet_ptl_attach_md+0x38e/0x540 [lnet] [ 965.080308] [] ? enqueue_task_fair+0x208/0x6c0 [ 965.080310] [] ? sched_clock_cpu+0xa5/0xe0 [ 965.080311] [] ? check_preempt_curr+0x80/0xa0 [ 965.080313] [] ? ttwu_do_wakeup+0x19/0x100 [ 965.080341] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 965.080368] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 965.080369] [] ? __getnstimeofday64+0x3f/0xd0 [ 965.080395] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 965.080420] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 965.080421] [] ? __wake_up_common_lock+0x91/0xc0 [ 965.080422] [] ? sched_feat_set+0xf0/0xf0 [ 965.080447] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 965.080450] [] ? __switch_to+0xce/0x5a0 [ 965.080475] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 965.080477] [] kthread+0xd1/0xe0 [ 965.080478] [] ? insert_kthread_work+0x40/0x40 [ 965.080480] [] ret_from_fork_nospec_begin+0x21/0x21 [ 965.080481] [] ? insert_kthread_work+0x40/0x40 [ 965.080481] Code: [ 965.080482] 0d [ 965.080482] 48 [ 965.080482] 98 [ 965.080483] 83 [ 965.080483] e2 [ 965.080483] 30 [ 965.080484] 48 [ 965.080484] 81 [ 965.080484] c2 [ 965.080484] c0 [ 965.080485] b8 [ 965.080485] 01 [ 965.080485] 00 [ 965.080485] 48 [ 965.080486] 03 [ 965.080486] 14 [ 965.080486] c5 [ 965.080486] e0 [ 965.080487] 17 [ 965.080487] d5 [ 965.080487] a0 [ 965.080487] 4c [ 965.080488] 89 [ 965.080488] 02 [ 965.080488] 41 [ 965.080488] 8b [ 965.080489] 40 [ 965.080489] 08 [ 965.080489] 85 [ 965.080490] c0 [ 965.080490] 75 [ 965.080490] 0f [ 965.080490] 0f [ 965.080491] 1f [ 965.080491] 44 [ 965.080491] 00 [ 965.080491] 00 [ 965.080492] f3 [ 965.080492] 90 [ 965.080492] 41 [ 965.080492] 8b [ 965.080493] 40 [ 965.080493] 08 [ 965.080493] <85> [ 965.080493] c0 [ 965.080494] 74 [ 965.080494] f6 [ 965.080494] 4d [ 965.080494] 8b [ 965.080495] 08 [ 965.080495] 4d [ 965.080495] 85 [ 965.080495] c9 [ 965.080496] 74 [ 965.080496] 04 [ 965.080496] 41 [ 965.080496] 0f [ 965.080497] 18 [ 965.080497] 09 [ 965.080497] 8b [ 965.080497] 17 [ 965.080498] 0f [ 965.080498] b7 [ 965.080498] c2 [ 965.080498] [ 965.088151] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [mdt00_002:10663] [ 965.088151] Modules linked in: [ 965.088152] osp(OE) [ 965.088152] mdd(OE) [ 965.088153] lod(OE) [ 965.088153] mdt(OE) [ 965.088154] lfsck(OE) [ 965.088154] mgs(OE) [ 965.088154] mgc(OE) [ 965.088155] osd_zfs(OE) [ 965.088155] lquota(OE) [ 965.088156] fid(OE) [ 965.088156] fld(OE) [ 965.088157] ptlrpc(OE) [ 965.088157] obdclass(OE) [ 965.088158] ko2iblnd(OE) [ 965.088158] lnet(OE) [ 965.088158] libcfs(OE) [ 965.088159] iTCO_wdt [ 965.088159] iTCO_vendor_support [ 965.088160] sb_edac [ 965.088160] intel_powerclamp [ 965.088161] coretemp [ 965.088161] intel_rapl [ 965.088161] iosf_mbi [ 965.088162] rpcrdma [ 965.088162] ib_iser [ 965.088162] kvm [ 965.088163] irqbypass [ 965.088163] joydev [ 965.088164] pcspkr [ 965.088164] lpc_ich [ 965.088164] i2c_i801 [ 965.088165] ioatdma [ 965.088165] ses [ 965.088166] enclosure [ 965.088166] sg [ 965.088166] ipmi_si [ 965.088167] ipmi_devintf [ 965.088167] ipmi_msghandler [ 965.088168] acpi_power_meter [ 965.088168] dm_round_robin [ 965.088169] acpi_cpufreq [ 965.088169] ib_ipoib [ 965.088169] rdma_ucm [ 965.088170] ib_umad [ 965.088170] sch_fq_codel [ 965.088171] iw_cxgb4 [ 965.088171] rdma_cm [ 965.088171] iw_cm [ 965.088172] ib_cm [ 965.088172] iw_cxgb3 [ 965.088173] zfs(POE) [ 965.088173] binfmt_misc [ 965.088174] zunicode(POE) [ 965.088174] zavl(POE) [ 965.088175] icp(POE) [ 965.088175] zcommon(POE) [ 965.088176] znvpair(POE) [ 965.088176] spl(OE) [ 965.088177] msr_safe(OE) [ 965.088177] ip_tables [ 965.088178] nfsv3 [ 965.088178] nfs_acl [ 965.088179] rpcsec_gss_krb5 [ 965.088179] auth_rpcgss [ 965.088179] nfsv4 [ 965.088180] dns_resolver [ 965.088180] nfs [ 965.088181] lockd [ 965.088181] grace [ 965.088181] fscache [ 965.088182] overlay(T) [ 965.088182] ext4 [ 965.088183] mbcache [ 965.088183] jbd2 [ 965.088184] dm_service_time [ 965.088184] sd_mod [ 965.088185] crc_t10dif [ 965.088185] crct10dif_generic [ 965.088185] mlx5_ib [ 965.088186] ib_uverbs [ 965.088186] ib_core [ 965.088187] be2iscsi [ 965.088187] bnx2i [ 965.088187] cnic [ 965.088188] uio [ 965.088188] cxgb4i [ 965.088188] cxgb4 [ 965.088189] cxgb3i [ 965.088189] cxgb3 [ 965.088190] mdio [ 965.088190] libcxgbi [ 965.088190] libcxgb [ 965.088191] qla4xxx [ 965.088191] iscsi_boot_sysfs [ 965.088192] mgag200 [ 965.088192] i2c_algo_bit [ 965.088192] 8021q [ 965.088193] drm_kms_helper [ 965.088193] garp [ 965.088194] syscopyarea [ 965.088194] mrp [ 965.088194] crct10dif_pclmul [ 965.088195] stp [ 965.088195] sysfillrect [ 965.088196] crct10dif_common [ 965.088196] sysimgblt [ 965.088196] crc32_pclmul [ 965.088197] llc [ 965.088197] fb_sys_fops [ 965.088198] crc32c_intel [ 965.088198] ttm [ 965.088198] ghash_clmulni_intel [ 965.088199] mlx5_core [ 965.088199] drm [ 965.088200] mxm_wmi [ 965.088200] ahci [ 965.088201] ixgbe(OE) [ 965.088201] aesni_intel [ 965.088201] mpt3sas [ 965.088202] mlxfw [ 965.088202] dm_multipath [ 965.088203] lrw [ 965.088203] devlink [ 965.088203] libahci [ 965.088204] gf128mul [ 965.088204] dca [ 965.088205] glue_helper [ 965.088205] ablk_helper [ 965.088205] raid_class [ 965.088206] ptp [ 965.088206] drm_panel_orientation_quirks [ 965.088207] libata [ 965.088207] cryptd [ 965.088208] scsi_transport_sas [ 965.088208] pps_core [ 965.088208] wmi [ 965.088209] sunrpc [ 965.088209] dm_mirror [ 965.088210] dm_region_hash [ 965.088210] dm_log [ 965.088210] dm_mod [ 965.088211] iscsi_tcp [ 965.088211] libiscsi_tcp [ 965.088212] libiscsi [ 965.088212] scsi_transport_iscsi [ 965.088213] fuse [ 965.088213] [ 965.088215] CPU: 5 PID: 10663 Comm: mdt00_002 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 965.088216] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 965.088217] task: ffff97f28e72a100 ti: ffff97f28e600000 task.ti: ffff97f28e600000 [ 965.088218] RIP: 0010:[] [ 965.088220] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 965.088221] RSP: 0018:ffff97f28e603ab0 EFLAGS: 00000246 [ 965.088222] RAX: 0000000000000000 RBX: ffff97f2bfb40000 RCX: 0000000000290000 [ 965.088223] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 965.088224] RBP: ffff97f28e603ab0 R08: ffff97f2bfb5b8c0 R09: 0000000000000000 [ 965.088225] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f28e603a78 [ 965.088226] R13: ffffffffa00ec9b1 R14: ffff97f28e603a58 R15: ffff97f28e603a88 [ 965.088227] FS: 0000000000000000(0000) GS:ffff97f2bfb40000(0000) knlGS:0000000000000000 [ 965.088228] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 965.088229] CR2: 00007ffff1096160 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 965.088230] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 965.088231] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 965.088231] Call Trace: [ 965.088234] [] queued_spin_lock_slowpath+0xb/0xf [ 965.088236] [] _raw_spin_lock+0x30/0x40 [ 965.088242] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 965.088248] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 965.088256] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 965.088258] [] ? wake_up_atomic_t+0x30/0x30 [ 965.088266] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 965.088291] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 965.088315] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 965.088341] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 965.088364] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 965.088387] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 965.088418] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 965.088446] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 965.088448] [] ? __getnstimeofday64+0x3f/0xd0 [ 965.088474] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 965.088498] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 965.088500] [] ? __wake_up_common_lock+0x91/0xc0 [ 965.088501] [] ? sched_feat_set+0xf0/0xf0 [ 965.088526] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 965.088528] [] ? __switch_to+0xce/0x5a0 [ 965.088552] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 965.088554] [] kthread+0xd1/0xe0 [ 965.088556] [] ? insert_kthread_work+0x40/0x40 [ 965.088557] [] ret_from_fork_nospec_begin+0x21/0x21 [ 965.088559] [] ? insert_kthread_work+0x40/0x40 [ 965.088560] Code: [ 965.088560] 0d [ 965.088561] 48 [ 965.088561] 98 [ 965.088561] 83 [ 965.088562] e2 [ 965.088562] 30 [ 965.088562] 48 [ 965.088563] 81 [ 965.088563] c2 [ 965.088563] c0 [ 965.088564] b8 [ 965.088564] 01 [ 965.088564] 00 [ 965.088565] 48 [ 965.088565] 03 [ 965.088565] 14 [ 965.088566] c5 [ 965.088566] e0 [ 965.088566] 17 [ 965.088567] d5 [ 965.088567] a0 [ 965.088567] 4c [ 965.088568] 89 [ 965.088568] 02 [ 965.088568] 41 [ 965.088569] 8b [ 965.088569] 40 [ 965.088569] 08 [ 965.088570] 85 [ 965.088570] c0 [ 965.088571] 75 [ 965.088571] 0f [ 965.088571] 0f [ 965.088572] 1f [ 965.088572] 44 [ 965.088572] 00 [ 965.088573] 00 [ 965.088573] f3 [ 965.088573] 90 [ 965.088574] 41 [ 965.088574] 8b [ 965.088575] 40 [ 965.088575] 08 [ 965.088576] <85> [ 965.088576] c0 [ 965.088576] 74 [ 965.088577] f6 [ 965.088577] 4d [ 965.088577] 8b [ 965.088578] 08 [ 965.088578] 4d [ 965.088578] 85 [ 965.088579] c9 [ 965.088579] 74 [ 965.088580] 04 [ 965.088580] 41 [ 965.088580] 0f [ 965.088581] 18 [ 965.088581] 09 [ 965.088581] 8b [ 965.088582] 17 [ 965.088582] 0f [ 965.088582] b7 [ 965.088583] c2 [ 965.088583] [ 965.195165] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [mdt01_003:10825] [ 965.195165] Modules linked in: [ 965.195166] osp(OE) [ 965.195166] mdd(OE) [ 965.195167] lod(OE) [ 965.195167] mdt(OE) [ 965.195168] lfsck(OE) [ 965.195168] mgs(OE) [ 965.195168] mgc(OE) [ 965.195169] osd_zfs(OE) [ 965.195169] lquota(OE) [ 965.195170] fid(OE) [ 965.195170] fld(OE) [ 965.195171] ptlrpc(OE) [ 965.195171] obdclass(OE) [ 965.195171] ko2iblnd(OE) [ 965.195172] lnet(OE) [ 965.195172] libcfs(OE) [ 965.195173] iTCO_wdt [ 965.195173] iTCO_vendor_support [ 965.195174] sb_edac [ 965.195174] intel_powerclamp [ 965.195175] coretemp [ 965.195175] intel_rapl [ 965.195175] iosf_mbi [ 965.195176] rpcrdma [ 965.195176] ib_iser [ 965.195177] kvm [ 965.195177] irqbypass [ 965.195178] joydev [ 965.195178] pcspkr [ 965.195178] lpc_ich [ 965.195179] i2c_i801 [ 965.195179] ioatdma [ 965.195179] ses [ 965.195180] enclosure [ 965.195180] sg [ 965.195181] ipmi_si [ 965.195181] ipmi_devintf [ 965.195181] ipmi_msghandler [ 965.195182] acpi_power_meter [ 965.195182] dm_round_robin [ 965.195183] acpi_cpufreq [ 965.195183] ib_ipoib [ 965.195184] rdma_ucm [ 965.195184] ib_umad [ 965.195184] sch_fq_codel [ 965.195185] iw_cxgb4 [ 965.195185] rdma_cm [ 965.195186] iw_cm [ 965.195186] ib_cm [ 965.195187] iw_cxgb3 [ 965.195187] zfs(POE) [ 965.195188] binfmt_misc [ 965.195188] zunicode(POE) [ 965.195189] zavl(POE) [ 965.195189] icp(POE) [ 965.195190] zcommon(POE) [ 965.195190] znvpair(POE) [ 965.195191] spl(OE) [ 965.195191] msr_safe(OE) [ 965.195192] ip_tables [ 965.195192] nfsv3 [ 965.195193] nfs_acl [ 965.195194] rpcsec_gss_krb5 [ 965.195194] auth_rpcgss [ 965.195195] nfsv4 [ 965.195195] dns_resolver [ 965.195196] nfs [ 965.195196] lockd [ 965.195197] grace [ 965.195197] fscache [ 965.195198] overlay(T) [ 965.195198] ext4 [ 965.195199] mbcache [ 965.195200] jbd2 [ 965.195200] dm_service_time [ 965.195201] sd_mod [ 965.195201] crc_t10dif [ 965.195201] crct10dif_generic [ 965.195202] mlx5_ib [ 965.195202] ib_uverbs [ 965.195203] ib_core [ 965.195203] be2iscsi [ 965.195204] bnx2i [ 965.195204] cnic [ 965.195205] uio [ 965.195205] cxgb4i [ 965.195206] cxgb4 [ 965.195206] cxgb3i [ 965.195206] cxgb3 [ 965.195207] mdio [ 965.195208] libcxgbi [ 965.195208] libcxgb [ 965.195209] qla4xxx [ 965.195209] iscsi_boot_sysfs [ 965.195209] mgag200 [ 965.195210] i2c_algo_bit [ 965.195210] 8021q [ 965.195211] drm_kms_helper [ 965.195212] garp [ 965.195212] syscopyarea [ 965.195213] mrp [ 965.195213] crct10dif_pclmul [ 965.195214] stp [ 965.195214] sysfillrect [ 965.195215] crct10dif_common [ 965.195216] sysimgblt [ 965.195216] crc32_pclmul [ 965.195217] llc [ 965.195217] fb_sys_fops [ 965.195218] crc32c_intel [ 965.195218] ttm [ 965.195218] ghash_clmulni_intel [ 965.195219] mlx5_core [ 965.195219] drm [ 965.195220] mxm_wmi [ 965.195220] ahci [ 965.195221] ixgbe(OE) [ 965.195221] aesni_intel [ 965.195222] mpt3sas [ 965.195222] mlxfw [ 965.195223] dm_multipath [ 965.195223] lrw [ 965.195223] devlink [ 965.195224] libahci [ 965.195224] gf128mul [ 965.195225] dca [ 965.195226] glue_helper [ 965.195226] ablk_helper [ 965.195227] raid_class [ 965.195227] ptp [ 965.195228] drm_panel_orientation_quirks [ 965.195228] libata [ 965.195229] cryptd [ 965.195229] scsi_transport_sas [ 965.195230] pps_core [ 965.195230] wmi [ 965.195231] sunrpc [ 965.195231] dm_mirror [ 965.195232] dm_region_hash [ 965.195232] dm_log [ 965.195232] dm_mod [ 965.195233] iscsi_tcp [ 965.195233] libiscsi_tcp [ 965.195234] libiscsi [ 965.195235] scsi_transport_iscsi [ 965.195235] fuse [ 965.195235] [ 965.195238] CPU: 13 PID: 10825 Comm: mdt01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 965.195239] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 965.195240] task: ffff98322385d280 ti: ffff983225500000 task.ti: ffff983225500000 [ 965.195240] RIP: 0010:[] [ 965.195244] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 965.195245] RSP: 0018:ffff983225503a50 EFLAGS: 00000246 [ 965.195246] RAX: 0000000000000000 RBX: ffff9832bf340000 RCX: 0000000000690000 [ 965.195247] RDX: ffff9832bf21b8c0 RSI: 0000000000410000 RDI: ffff97f2a83bcf40 [ 965.195248] RBP: ffff983225503a50 R08: ffff9832bf35b8c0 R09: 0000000000000000 [ 965.195248] R10: 0000000000000001 R11: 0000000000000005 R12: ffff983225503a18 [ 965.195249] R13: ffffffffa00ec9b1 R14: ffff9832255039f8 R15: ffff983225503a28 [ 965.195251] FS: 0000000000000000(0000) GS:ffff9832bf340000(0000) knlGS:0000000000000000 [ 965.195251] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 965.195252] CR2: 00002aaaab176a00 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 965.195253] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 965.195254] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 965.195254] Call Trace: [ 965.195259] [] queued_spin_lock_slowpath+0xb/0xf [ 965.195261] [] _raw_spin_lock+0x30/0x40 [ 965.195268] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 965.195286] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 965.195288] [] ? wake_up_atomic_t+0x30/0x30 [ 965.195296] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 965.195325] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 965.195349] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 965.195352] [] ? enqueue_task_fair+0x208/0x6c0 [ 965.195353] [] ? check_preempt_curr+0x80/0xa0 [ 965.195355] [] ? ttwu_do_wakeup+0x19/0x100 [ 965.195387] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 965.195415] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 965.195417] [] ? __getnstimeofday64+0x3f/0xd0 [ 965.195443] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 965.195468] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 965.195469] [] ? __wake_up_common_lock+0x91/0xc0 [ 965.195471] [] ? sched_feat_set+0xf0/0xf0 [ 965.195496] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 965.195520] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 965.195522] [] kthread+0xd1/0xe0 [ 965.195524] [] ? insert_kthread_work+0x40/0x40 [ 965.195526] [] ret_from_fork_nospec_begin+0x21/0x21 [ 965.195528] [] ? insert_kthread_work+0x40/0x40 [ 965.195528] Code: [ 965.195529] 0d [ 965.195529] 48 [ 965.195529] 98 [ 965.195530] 83 [ 965.195530] e2 [ 965.195530] 30 [ 965.195531] 48 [ 965.195531] 81 [ 965.195531] c2 [ 965.195532] c0 [ 965.195532] b8 [ 965.195533] 01 [ 965.195533] 00 [ 965.195533] 48 [ 965.195534] 03 [ 965.195534] 14 [ 965.195534] c5 [ 965.195535] e0 [ 965.195535] 17 [ 965.195535] d5 [ 965.195536] a0 [ 965.195536] 4c [ 965.195536] 89 [ 965.195537] 02 [ 965.195537] 41 [ 965.195538] 8b [ 965.195538] 40 [ 965.195538] 08 [ 965.195539] 85 [ 965.195539] c0 [ 965.195539] 75 [ 965.195540] 0f [ 965.195540] 0f [ 965.195541] 1f [ 965.195541] 44 [ 965.195541] 00 [ 965.195542] 00 [ 965.195542] f3 [ 965.195542] 90 [ 965.195543] 41 [ 965.195543] 8b [ 965.195544] 40 [ 965.195544] 08 [ 965.195544] <85> [ 965.195545] c0 [ 965.195545] 74 [ 965.195546] f6 [ 965.195546] 4d [ 965.195546] 8b [ 965.195547] 08 [ 965.195547] 4d [ 965.195547] 85 [ 965.195548] c9 [ 965.195548] 74 [ 965.195548] 04 [ 965.195549] 41 [ 965.195549] 0f [ 965.195550] 18 [ 965.195550] 09 [ 965.195550] 8b [ 965.195551] 17 [ 965.195551] 0f [ 965.195551] b7 [ 965.195552] c2 [ 965.195552] [ 966.613293] lod(OE) [ 966.616223] mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 [ 966.694391] cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 966.751009] CPU: 0 PID: 17320 Comm: mdt00_008 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 966.765274] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 966.777602] task: ffff98321989e300 ti: ffff98321cd1c000 task.ti: ffff98321cd1c000 [ 966.786437] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 966.797999] RSP: 0018:ffff98321cd1fab0 EFLAGS: 00000246 [ 966.804409] RAX: 0000000000000000 RBX: ffff97f2bfa00000 RCX: 0000000000010000 [ 966.812857] RDX: ffff9832bf4db8c0 RSI: 0000000000d90000 RDI: ffff97f2a83bcf40 [ 966.821305] RBP: ffff98321cd1fab0 R08: ffff97f2bfa1b8c0 R09: 0000000000000000 [ 966.829751] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321cd1fa78 [ 966.838200] R13: ffffffffa00ec9b1 R14: ffff98321cd1fa58 R15: ffff98321cd1fa88 [ 966.846647] FS: 0000000000000000(0000) GS:ffff97f2bfa00000(0000) knlGS:0000000000000000 [ 966.856161] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 966.863058] CR2: 00007ffff7ff8000 CR3: 0000007f0e586000 CR4: 00000000003607f0 [ 966.871505] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 966.879953] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 966.888401] Call Trace: [ 966.891613] [] queued_spin_lock_slowpath+0xb/0xf [ 966.899093] [] _raw_spin_lock+0x30/0x40 [ 966.905706] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 966.913381] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 966.921350] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 966.929896] [] ? wake_up_atomic_t+0x30/0x30 [ 966.936895] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 966.944303] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 966.952581] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 966.960546] [] ? check_preempt_curr+0x80/0xa0 [ 966.967756] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 966.975646] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 966.982961] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 966.991148] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 967.000082] [] ? __getnstimeofday64+0x3f/0xd0 [ 967.007293] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 967.016346] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 967.024408] [] ? __wake_up_common_lock+0x91/0xc0 [ 967.031886] [] ? sched_feat_set+0xf0/0xf0 [ 967.038709] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 967.046190] [] ? __switch_to+0xce/0x5a0 [ 967.052818] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 967.061558] [] kthread+0xd1/0xe0 [ 967.067485] [] ? insert_kthread_work+0x40/0x40 [ 967.074771] [] ret_from_fork_nospec_begin+0x21/0x21 [ 967.082540] [] ? insert_kthread_work+0x40/0x40 [ 967.089824] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 973.223087] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [mdt00_009:17321] [ 973.232214] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 973.312020] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 973.373069] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 973.387430] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 973.399757] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 973.408591] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 973.420152] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 973.426564] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 973.435012] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 973.443460] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 973.451907] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 973.460353] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 973.468800] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 973.478314] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 973.485211] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 973.493658] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 973.502106] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 973.510554] Call Trace: [ 973.513768] [] queued_spin_lock_slowpath+0xb/0xf [ 973.521248] [] _raw_spin_lock+0x30/0x40 [ 973.527865] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 973.535544] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 973.543994] [] ? wake_up_atomic_t+0x30/0x30 [ 973.550995] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 973.558409] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 973.566686] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 973.575254] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 973.583412] [] ? enqueue_task_fair+0x208/0x6c0 [ 973.590697] [] ? sched_clock_cpu+0xa5/0xe0 [ 973.597595] [] ? check_preempt_curr+0x80/0xa0 [ 973.604782] [] ? ttwu_do_wakeup+0x19/0x100 [ 973.611710] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 973.619895] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 973.628829] [] ? __getnstimeofday64+0x3f/0xd0 [ 973.636042] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 973.645096] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 973.653156] [] ? __wake_up_common_lock+0x91/0xc0 [ 973.660634] [] ? sched_feat_set+0xf0/0xf0 [ 973.667457] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 973.674937] [] ? __switch_to+0xce/0x5a0 [ 973.681568] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 973.690309] [] kthread+0xd1/0xe0 [ 973.696238] [] ? insert_kthread_work+0x40/0x40 [ 973.703523] [] ret_from_fork_nospec_begin+0x21/0x21 [ 973.711293] [] ? insert_kthread_work+0x40/0x40 [ 973.718575] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 977.077531] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [mdt00_013:17594] [ 977.086552] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 977.174542] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [mdt01_022:17601] [ 977.166396] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx [ 977.174544] Modules linked in: [ 977.174545] osp(OE) [ 977.174546] mdd(OE) [ 977.174546] lod(OE) [ 977.174547] mdt(OE) [ 977.174548] lfsck(OE) [ 977.174548] mgs(OE) [ 977.174548] mgc(OE) [ 977.174549] osd_zfs(OE) [ 977.174549] lquota(OE) [ 977.174550] fid(OE) [ 977.174550] fld(OE) [ 977.174551] ptlrpc(OE) [ 977.174551] obdclass(OE) [ 977.174551] ko2iblnd(OE) [ 977.174552] lnet(OE) [ 977.174552] libcfs(OE) [ 977.174552] iTCO_wdt [ 977.174553] iTCO_vendor_support [ 977.174553] sb_edac [ 977.174554] intel_powerclamp [ 977.174554] coretemp [ 977.174555] intel_rapl [ 977.174555] iosf_mbi [ 977.174556] rpcrdma [ 977.174556] ib_iser [ 977.174557] kvm [ 977.174557] irqbypass [ 977.174558] joydev [ 977.174558] pcspkr [ 977.174558] lpc_ich [ 977.174559] i2c_i801 [ 977.174559] ioatdma [ 977.174560] ses [ 977.174560] enclosure [ 977.174561] sg [ 977.174561] ipmi_si [ 977.174562] ipmi_devintf [ 977.174562] ipmi_msghandler [ 977.174563] acpi_power_meter [ 977.174563] dm_round_robin [ 977.174563] acpi_cpufreq [ 977.174564] ib_ipoib [ 977.174564] rdma_ucm [ 977.174565] ib_umad [ 977.174565] sch_fq_codel [ 977.174566] iw_cxgb4 [ 977.174566] rdma_cm [ 977.174567] iw_cm [ 977.174567] ib_cm [ 977.174567] iw_cxgb3 [ 977.174568] zfs(POE) [ 977.174569] binfmt_misc [ 977.174569] zunicode(POE) [ 977.174569] zavl(POE) [ 977.174570] icp(POE) [ 977.174570] zcommon(POE) [ 977.174571] znvpair(POE) [ 977.174571] spl(OE) [ 977.174572] msr_safe(OE) [ 977.174572] ip_tables [ 977.174572] nfsv3 [ 977.174573] nfs_acl [ 977.174573] rpcsec_gss_krb5 [ 977.174574] auth_rpcgss [ 977.174574] nfsv4 [ 977.174575] dns_resolver [ 977.174575] nfs [ 977.174576] lockd [ 977.174576] grace [ 977.174577] fscache [ 977.174577] overlay(T) [ 977.174578] ext4 [ 977.174578] mbcache [ 977.174579] jbd2 [ 977.174579] dm_service_time [ 977.174579] sd_mod [ 977.174580] crc_t10dif [ 977.174580] crct10dif_generic [ 977.174581] mlx5_ib [ 977.174581] ib_uverbs [ 977.174582] ib_core [ 977.174582] be2iscsi [ 977.174583] bnx2i [ 977.174583] cnic [ 977.174584] uio [ 977.174584] cxgb4i [ 977.174584] cxgb4 [ 977.174585] cxgb3i [ 977.174585] cxgb3 [ 977.174586] mdio [ 977.174586] libcxgbi [ 977.174587] libcxgb [ 977.174587] qla4xxx [ 977.174587] iscsi_boot_sysfs [ 977.174588] mgag200 [ 977.174588] i2c_algo_bit [ 977.174589] 8021q [ 977.174589] drm_kms_helper [ 977.174590] garp [ 977.174590] syscopyarea [ 977.174591] mrp [ 977.174591] crct10dif_pclmul [ 977.174592] stp [ 977.174592] sysfillrect [ 977.174593] crct10dif_common [ 977.174593] sysimgblt [ 977.174593] crc32_pclmul [ 977.174594] llc [ 977.174594] fb_sys_fops [ 977.174595] crc32c_intel [ 977.174595] ttm [ 977.174595] ghash_clmulni_intel [ 977.174596] mlx5_core [ 977.174596] drm [ 977.174597] mxm_wmi [ 977.174597] ahci [ 977.174597] ixgbe(OE) [ 977.174598] aesni_intel [ 977.174598] mpt3sas [ 977.174599] mlxfw [ 977.174599] dm_multipath [ 977.174599] lrw [ 977.174600] devlink [ 977.174600] libahci [ 977.174601] gf128mul [ 977.174601] dca [ 977.174601] glue_helper [ 977.174602] ablk_helper [ 977.174602] raid_class [ 977.174603] ptp [ 977.174603] drm_panel_orientation_quirks [ 977.174603] libata [ 977.174604] cryptd [ 977.174604] scsi_transport_sas [ 977.174605] pps_core [ 977.174605] wmi [ 977.174605] sunrpc [ 977.174606] dm_mirror [ 977.174606] dm_region_hash [ 977.174607] dm_log [ 977.174607] dm_mod [ 977.174608] iscsi_tcp [ 977.174608] libiscsi_tcp [ 977.174608] libiscsi [ 977.174609] scsi_transport_iscsi [ 977.174609] fuse [ 977.174610] [ 977.174613] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.174613] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.174614] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 977.174615] RIP: 0010:[] [ 977.174620] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 977.174621] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 977.174622] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 977.174622] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 977.174623] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 977.174624] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 977.174624] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 977.174625] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 977.174626] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.174627] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 977.174628] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.174629] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.174629] Call Trace: [ 977.174635] [] queued_spin_lock_slowpath+0xb/0xf [ 977.174638] [] _raw_spin_lock+0x30/0x40 [ 977.174648] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.174664] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.174667] [] ? wake_up_atomic_t+0x30/0x30 [ 977.174673] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.174708] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.174727] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 977.174746] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 977.174764] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 977.174781] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 977.174808] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 977.174835] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.174837] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.174863] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.174888] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.174890] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.174891] [] ? sched_feat_set+0xf0/0xf0 [ 977.174916] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.174942] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.174944] [] kthread+0xd1/0xe0 [ 977.174945] [] ? insert_kthread_work+0x40/0x40 [ 977.174947] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.174949] [] ? insert_kthread_work+0x40/0x40 [ 977.174949] Code: [ 977.174950] 0d [ 977.174950] 48 [ 977.174950] 98 [ 977.174950] 83 [ 977.174951] e2 [ 977.174951] 30 [ 977.174951] 48 [ 977.174951] 81 [ 977.174952] c2 [ 977.174952] c0 [ 977.174952] b8 [ 977.174952] 01 [ 977.174953] 00 [ 977.174953] 48 [ 977.174953] 03 [ 977.174953] 14 [ 977.174954] c5 [ 977.174954] e0 [ 977.174954] 17 [ 977.174954] d5 [ 977.174955] a0 [ 977.174955] 4c [ 977.174955] 89 [ 977.174955] 02 [ 977.174956] 41 [ 977.174956] 8b [ 977.174956] 40 [ 977.174956] 08 [ 977.174957] 85 [ 977.174957] c0 [ 977.174957] 75 [ 977.174958] 0f [ 977.174958] 0f [ 977.174958] 1f [ 977.174958] 44 [ 977.174958] 00 [ 977.174959] 00 [ 977.174959] f3 [ 977.174959] 90 [ 977.174960] 41 [ 977.174960] 8b [ 977.174960] 40 [ 977.174960] 08 [ 977.174961] <85> [ 977.174961] c0 [ 977.174961] 74 [ 977.174961] f6 [ 977.174962] 4d [ 977.174962] 8b [ 977.174962] 08 [ 977.174963] 4d [ 977.174963] 85 [ 977.174963] c9 [ 977.174963] 74 [ 977.174964] 04 [ 977.174964] 41 [ 977.174964] 0f [ 977.174964] 18 [ 977.174965] 09 [ 977.174965] 8b [ 977.174965] 17 [ 977.174965] 0f [ 977.174966] b7 [ 977.174966] c2 [ 977.174966] [ 977.201544] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [mdt01_016:17554] [ 977.201545] Modules linked in: [ 977.201545] osp(OE) [ 977.201546] mdd(OE) [ 977.201546] lod(OE) [ 977.201547] mdt(OE) [ 977.201547] lfsck(OE) [ 977.201547] mgs(OE) [ 977.201548] mgc(OE) [ 977.201548] osd_zfs(OE) [ 977.201549] lquota(OE) [ 977.201549] fid(OE) [ 977.201549] fld(OE) [ 977.201550] ptlrpc(OE) [ 977.201550] obdclass(OE) [ 977.201551] ko2iblnd(OE) [ 977.201551] lnet(OE) [ 977.201552] libcfs(OE) [ 977.201552] iTCO_wdt [ 977.201552] iTCO_vendor_support [ 977.201553] sb_edac [ 977.201553] intel_powerclamp [ 977.201554] coretemp [ 977.201554] intel_rapl [ 977.201554] iosf_mbi [ 977.201555] rpcrdma [ 977.201555] ib_iser [ 977.201555] kvm [ 977.201556] irqbypass [ 977.201556] joydev [ 977.201557] pcspkr [ 977.201557] lpc_ich [ 977.201557] i2c_i801 [ 977.201558] ioatdma [ 977.201558] ses [ 977.201558] enclosure [ 977.201559] sg [ 977.201559] ipmi_si [ 977.201559] ipmi_devintf [ 977.201560] ipmi_msghandler [ 977.201560] acpi_power_meter [ 977.201561] dm_round_robin [ 977.201561] acpi_cpufreq [ 977.201561] ib_ipoib [ 977.201562] rdma_ucm [ 977.201562] ib_umad [ 977.201562] sch_fq_codel [ 977.201563] iw_cxgb4 [ 977.201563] rdma_cm [ 977.201564] iw_cm [ 977.201564] ib_cm [ 977.201564] iw_cxgb3 [ 977.201565] zfs(POE) [ 977.201565] binfmt_misc [ 977.201566] zunicode(POE) [ 977.201566] zavl(POE) [ 977.201566] icp(POE) [ 977.201567] zcommon(POE) [ 977.201567] znvpair(POE) [ 977.201568] spl(OE) [ 977.201568] msr_safe(OE) [ 977.201569] ip_tables [ 977.201569] nfsv3 [ 977.201569] nfs_acl [ 977.201570] rpcsec_gss_krb5 [ 977.201570] auth_rpcgss [ 977.201570] nfsv4 [ 977.201571] dns_resolver [ 977.201571] nfs [ 977.201572] lockd [ 977.201572] grace [ 977.201572] fscache [ 977.201573] overlay(T) [ 977.201573] ext4 [ 977.201574] mbcache [ 977.201574] jbd2 [ 977.201575] dm_service_time [ 977.201575] sd_mod [ 977.201575] crc_t10dif [ 977.201576] crct10dif_generic [ 977.201576] mlx5_ib [ 977.201577] ib_uverbs [ 977.201577] ib_core [ 977.201578] be2iscsi [ 977.201578] bnx2i [ 977.201578] cnic [ 977.201579] uio [ 977.201579] cxgb4i [ 977.201580] cxgb4 [ 977.201580] cxgb3i [ 977.201580] cxgb3 [ 977.201581] mdio [ 977.201581] libcxgbi [ 977.201582] libcxgb [ 977.201582] qla4xxx [ 977.201582] iscsi_boot_sysfs [ 977.201583] mgag200 [ 977.201583] i2c_algo_bit [ 977.201584] 8021q [ 977.201584] drm_kms_helper [ 977.201584] garp [ 977.201585] syscopyarea [ 977.201585] mrp [ 977.201586] crct10dif_pclmul [ 977.201586] stp [ 977.201586] sysfillrect [ 977.201587] crct10dif_common [ 977.201587] sysimgblt [ 977.201588] crc32_pclmul [ 977.201588] llc [ 977.201589] fb_sys_fops [ 977.201589] crc32c_intel [ 977.201589] ttm [ 977.201590] ghash_clmulni_intel [ 977.201590] mlx5_core [ 977.201591] drm [ 977.201591] mxm_wmi [ 977.201591] ahci [ 977.201592] ixgbe(OE) [ 977.201592] aesni_intel [ 977.201593] mpt3sas [ 977.201593] mlxfw [ 977.201593] dm_multipath [ 977.201594] lrw [ 977.201594] devlink [ 977.201595] libahci [ 977.201595] gf128mul [ 977.201595] dca [ 977.201596] glue_helper [ 977.201596] ablk_helper [ 977.201596] raid_class [ 977.201597] ptp [ 977.201597] drm_panel_orientation_quirks [ 977.201598] libata [ 977.201598] cryptd [ 977.201598] scsi_transport_sas [ 977.201599] pps_core [ 977.201599] wmi [ 977.201600] sunrpc [ 977.201600] dm_mirror [ 977.201600] dm_region_hash [ 977.201601] dm_log [ 977.201601] dm_mod [ 977.201602] iscsi_tcp [ 977.201602] libiscsi_tcp [ 977.201602] libiscsi [ 977.201603] scsi_transport_iscsi [ 977.201603] fuse [ 977.201603] [ 977.201605] CPU: 14 PID: 17554 Comm: mdt01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.201606] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.201607] task: ffff98321c985280 ti: ffff98321b544000 task.ti: ffff98321b544000 [ 977.201608] RIP: 0010:[] [ 977.201610] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 977.201611] RSP: 0018:ffff98321b547ab0 EFLAGS: 00000246 [ 977.201612] RAX: 0000000000000000 RBX: ffff9832bf380000 RCX: 0000000000710000 [ 977.201612] RDX: ffff97f2bfd1b8c0 RSI: 0000000000a10000 RDI: ffff97f2a83bcf40 [ 977.201613] RBP: ffff98321b547ab0 R08: ffff9832bf39b8c0 R09: 0000000000000000 [ 977.201614] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321b547a78 [ 977.201615] R13: ffffffffa00ec9b1 R14: ffff98321b547a58 R15: ffff98321b547a88 [ 977.201616] FS: 0000000000000000(0000) GS:ffff9832bf380000(0000) knlGS:0000000000000000 [ 977.201617] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.201618] CR2: 00007ffff7fd6000 CR3: 0000003f67e7a000 CR4: 00000000003607e0 [ 977.201619] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.201620] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.201620] Call Trace: [ 977.201622] [] queued_spin_lock_slowpath+0xb/0xf [ 977.201624] [] _raw_spin_lock+0x30/0x40 [ 977.201631] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.201639] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.201641] [] ? wake_up_atomic_t+0x30/0x30 [ 977.201648] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.201674] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.201700] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 977.201702] [] ? check_preempt_curr+0x80/0xa0 [ 977.201726] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 977.201748] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 977.201778] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 977.201806] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.201809] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.201835] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.201859] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.201861] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.201862] [] ? sched_feat_set+0xf0/0xf0 [ 977.201887] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.201911] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.201913] [] kthread+0xd1/0xe0 [ 977.201915] [] ? insert_kthread_work+0x40/0x40 [ 977.201916] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.201918] [] ? insert_kthread_work+0x40/0x40 [ 977.201919] Code: [ 977.201919] 0d [ 977.201920] 48 [ 977.201920] 98 [ 977.201920] 83 [ 977.201921] e2 [ 977.201921] 30 [ 977.201921] 48 [ 977.201922] 81 [ 977.201922] c2 [ 977.201922] c0 [ 977.201923] b8 [ 977.201923] 01 [ 977.201924] 00 [ 977.201924] 48 [ 977.201924] 03 [ 977.201925] 14 [ 977.201925] c5 [ 977.201925] e0 [ 977.201926] 17 [ 977.201926] d5 [ 977.201927] a0 [ 977.201927] 4c [ 977.201927] 89 [ 977.201928] 02 [ 977.201928] 41 [ 977.201928] 8b [ 977.201929] 40 [ 977.201929] 08 [ 977.201930] 85 [ 977.201930] c0 [ 977.201931] 75 [ 977.201931] 0f [ 977.201931] 0f [ 977.201932] 1f [ 977.201932] 44 [ 977.201933] 00 [ 977.201933] 00 [ 977.201933] f3 [ 977.201934] 90 [ 977.201934] 41 [ 977.201934] 8b [ 977.201935] 40 [ 977.201935] 08 [ 977.201935] <85> [ 977.201936] c0 [ 977.201936] 74 [ 977.201936] f6 [ 977.201937] 4d [ 977.201937] 8b [ 977.201937] 08 [ 977.201938] 4d [ 977.201938] 85 [ 977.201938] c9 [ 977.201939] 74 [ 977.201939] 04 [ 977.201939] 41 [ 977.201940] 0f [ 977.201940] 18 [ 977.201941] 09 [ 977.201941] 8b [ 977.201941] 17 [ 977.201942] 0f [ 977.201942] b7 [ 977.201942] c2 [ 977.201942] [ 977.205548] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [mdt01_007:17310] [ 977.205548] Modules linked in: [ 977.205548] osp(OE) [ 977.205549] mdd(OE) [ 977.205549] lod(OE) [ 977.205549] mdt(OE) [ 977.205550] lfsck(OE) [ 977.205550] mgs(OE) [ 977.205550] mgc(OE) [ 977.205551] osd_zfs(OE) [ 977.205551] lquota(OE) [ 977.205551] fid(OE) [ 977.205552] fld(OE) [ 977.205552] ptlrpc(OE) [ 977.205552] obdclass(OE) [ 977.205552] ko2iblnd(OE) [ 977.205553] lnet(OE) [ 977.205553] libcfs(OE) [ 977.205553] iTCO_wdt [ 977.205554] iTCO_vendor_support [ 977.205554] sb_edac [ 977.205554] intel_powerclamp [ 977.205554] coretemp [ 977.205555] intel_rapl [ 977.205555] iosf_mbi [ 977.205555] rpcrdma [ 977.205556] ib_iser [ 977.205556] kvm [ 977.205556] irqbypass [ 977.205556] joydev [ 977.205557] pcspkr [ 977.205557] lpc_ich [ 977.205557] i2c_i801 [ 977.205557] ioatdma [ 977.205558] ses [ 977.205558] enclosure [ 977.205558] sg [ 977.205558] ipmi_si [ 977.205559] ipmi_devintf [ 977.205559] ipmi_msghandler [ 977.205559] acpi_power_meter [ 977.205560] dm_round_robin [ 977.205560] acpi_cpufreq [ 977.205560] ib_ipoib [ 977.205560] rdma_ucm [ 977.205561] ib_umad [ 977.205561] sch_fq_codel [ 977.205561] iw_cxgb4 [ 977.205561] rdma_cm [ 977.205562] iw_cm [ 977.205562] ib_cm [ 977.205562] iw_cxgb3 [ 977.205563] zfs(POE) [ 977.205563] binfmt_misc [ 977.205563] zunicode(POE) [ 977.205564] zavl(POE) [ 977.205564] icp(POE) [ 977.205564] zcommon(POE) [ 977.205565] znvpair(POE) [ 977.205565] spl(OE) [ 977.205565] msr_safe(OE) [ 977.205565] ip_tables [ 977.205566] nfsv3 [ 977.205566] nfs_acl [ 977.205566] rpcsec_gss_krb5 [ 977.205567] auth_rpcgss [ 977.205567] nfsv4 [ 977.205567] dns_resolver [ 977.205567] nfs [ 977.205568] lockd [ 977.205568] grace [ 977.205568] fscache [ 977.205569] overlay(T) [ 977.205569] ext4 [ 977.205569] mbcache [ 977.205570] jbd2 [ 977.205570] dm_service_time [ 977.205570] sd_mod [ 977.205571] crc_t10dif [ 977.205571] crct10dif_generic [ 977.205571] mlx5_ib [ 977.205571] ib_uverbs [ 977.205572] ib_core [ 977.205572] be2iscsi [ 977.205572] bnx2i [ 977.205572] cnic [ 977.205573] uio [ 977.205573] cxgb4i [ 977.205573] cxgb4 [ 977.205574] cxgb3i [ 977.205574] cxgb3 [ 977.205574] mdio [ 977.205574] libcxgbi [ 977.205575] libcxgb [ 977.205575] qla4xxx [ 977.205575] iscsi_boot_sysfs [ 977.205576] mgag200 [ 977.205576] i2c_algo_bit [ 977.205576] 8021q [ 977.205577] drm_kms_helper [ 977.205577] garp [ 977.205577] syscopyarea [ 977.205577] mrp [ 977.205578] crct10dif_pclmul [ 977.205578] stp [ 977.205578] sysfillrect [ 977.205579] crct10dif_common [ 977.205579] sysimgblt [ 977.205579] crc32_pclmul [ 977.205580] llc [ 977.205580] fb_sys_fops [ 977.205580] crc32c_intel [ 977.205581] ttm [ 977.205581] ghash_clmulni_intel [ 977.205581] mlx5_core [ 977.205581] drm [ 977.205582] mxm_wmi [ 977.205582] ahci [ 977.205582] ixgbe(OE) [ 977.205583] aesni_intel [ 977.205583] mpt3sas [ 977.205583] mlxfw [ 977.205584] dm_multipath [ 977.205584] lrw [ 977.205584] devlink [ 977.205584] libahci [ 977.205585] gf128mul [ 977.205585] dca [ 977.205585] glue_helper [ 977.205586] ablk_helper [ 977.205586] raid_class [ 977.205586] ptp [ 977.205587] drm_panel_orientation_quirks [ 977.205587] libata [ 977.205587] cryptd [ 977.205587] scsi_transport_sas [ 977.205588] pps_core [ 977.205588] wmi [ 977.205588] sunrpc [ 977.205589] dm_mirror [ 977.205589] dm_region_hash [ 977.205589] dm_log [ 977.205590] dm_mod [ 977.205590] iscsi_tcp [ 977.205590] libiscsi_tcp [ 977.205591] libiscsi [ 977.205591] scsi_transport_iscsi [ 977.205591] fuse [ 977.205592] [ 977.205593] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.205594] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.205594] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 977.205595] RIP: 0010:[] [ 977.205597] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 977.205598] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 977.205599] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 977.205599] RDX: ffff97f2bfc5b8c0 RSI: 0000000000890000 RDI: ffff97f2a83bcf40 [ 977.205600] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 977.205600] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 977.205601] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 977.205602] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 977.205603] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.205603] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 977.205604] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.205604] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.205605] Call Trace: [ 977.205607] [] queued_spin_lock_slowpath+0xb/0xf [ 977.205609] [] _raw_spin_lock+0x30/0x40 [ 977.205614] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.205620] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.205622] [] ? wake_up_atomic_t+0x30/0x30 [ 977.205627] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.205646] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.205664] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 977.205665] [] ? check_preempt_curr+0x80/0xa0 [ 977.205683] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 977.205699] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 977.205722] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 977.205744] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.205745] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.205765] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.205783] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.205784] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.205785] [] ? sched_feat_set+0xf0/0xf0 [ 977.205804] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.205822] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.205824] [] kthread+0xd1/0xe0 [ 977.205825] [] ? insert_kthread_work+0x40/0x40 [ 977.205827] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.205828] [] ? insert_kthread_work+0x40/0x40 [ 977.205828] Code: [ 977.205829] 0d [ 977.205829] 48 [ 977.205829] 98 [ 977.205830] 83 [ 977.205830] e2 [ 977.205830] 30 [ 977.205831] 48 [ 977.205831] 81 [ 977.205831] c2 [ 977.205831] c0 [ 977.205832] b8 [ 977.205832] 01 [ 977.205832] 00 [ 977.205832] 48 [ 977.205833] 03 [ 977.205833] 14 [ 977.205833] c5 [ 977.205833] e0 [ 977.205834] 17 [ 977.205834] d5 [ 977.205834] a0 [ 977.205835] 4c [ 977.205835] 89 [ 977.205835] 02 [ 977.205835] 41 [ 977.205836] 8b [ 977.205836] 40 [ 977.205836] 08 [ 977.205836] 85 [ 977.205837] c0 [ 977.205837] 75 [ 977.205837] 0f [ 977.205838] 0f [ 977.205838] 1f [ 977.205838] 44 [ 977.205838] 00 [ 977.205839] 00 [ 977.205839] f3 [ 977.205839] 90 [ 977.205840] 41 [ 977.205840] 8b [ 977.205840] 40 [ 977.205840] 08 [ 977.205841] <85> [ 977.205841] c0 [ 977.205841] 74 [ 977.205841] f6 [ 977.205842] 4d [ 977.205842] 8b [ 977.205842] 08 [ 977.205842] 4d [ 977.205843] 85 [ 977.205843] c9 [ 977.205843] 74 [ 977.205843] 04 [ 977.205844] 41 [ 977.205844] 0f [ 977.205844] 18 [ 977.205844] 09 [ 977.205845] 8b [ 977.205845] 17 [ 977.205845] 0f [ 977.205845] b7 [ 977.205846] c2 [ 977.205846] [ 977.219547] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ll_mgs_0013:10822] [ 977.219548] Modules linked in: [ 977.219548] osp(OE) [ 977.219549] mdd(OE) [ 977.219549] lod(OE) [ 977.219549] mdt(OE) [ 977.219550] lfsck(OE) [ 977.219550] mgs(OE) [ 977.219550] mgc(OE) [ 977.219551] osd_zfs(OE) [ 977.219551] lquota(OE) [ 977.219551] fid(OE) [ 977.219551] fld(OE) [ 977.219552] ptlrpc(OE) [ 977.219552] obdclass(OE) [ 977.219552] ko2iblnd(OE) [ 977.219553] lnet(OE) [ 977.219553] libcfs(OE) [ 977.219553] iTCO_wdt [ 977.219554] iTCO_vendor_support [ 977.219554] sb_edac [ 977.219554] intel_powerclamp [ 977.219554] coretemp [ 977.219555] intel_rapl [ 977.219555] iosf_mbi [ 977.219555] rpcrdma [ 977.219555] ib_iser [ 977.219556] kvm [ 977.219556] irqbypass [ 977.219556] joydev [ 977.219556] pcspkr [ 977.219557] lpc_ich [ 977.219557] i2c_i801 [ 977.219557] ioatdma [ 977.219558] ses [ 977.219558] enclosure [ 977.219558] sg [ 977.219558] ipmi_si [ 977.219559] ipmi_devintf [ 977.219559] ipmi_msghandler [ 977.219559] acpi_power_meter [ 977.219559] dm_round_robin [ 977.219560] acpi_cpufreq [ 977.219560] ib_ipoib [ 977.219560] rdma_ucm [ 977.219561] ib_umad [ 977.219561] sch_fq_codel [ 977.219561] iw_cxgb4 [ 977.219561] rdma_cm [ 977.219562] iw_cm [ 977.219562] ib_cm [ 977.219562] iw_cxgb3 [ 977.219563] zfs(POE) [ 977.219563] binfmt_misc [ 977.219563] zunicode(POE) [ 977.219563] zavl(POE) [ 977.219564] icp(POE) [ 977.219564] zcommon(POE) [ 977.219564] znvpair(POE) [ 977.219565] spl(OE) [ 977.219565] msr_safe(OE) [ 977.219565] ip_tables [ 977.219566] nfsv3 [ 977.219566] nfs_acl [ 977.219566] rpcsec_gss_krb5 [ 977.219567] auth_rpcgss [ 977.219567] nfsv4 [ 977.219567] dns_resolver [ 977.219567] nfs [ 977.219568] lockd [ 977.219568] grace [ 977.219568] fscache [ 977.219569] overlay(T) [ 977.219569] ext4 [ 977.219569] mbcache [ 977.219570] jbd2 [ 977.219570] dm_service_time [ 977.219570] sd_mod [ 977.219570] crc_t10dif [ 977.219571] crct10dif_generic [ 977.219571] mlx5_ib [ 977.219571] ib_uverbs [ 977.219572] ib_core [ 977.219572] be2iscsi [ 977.219572] bnx2i [ 977.219573] cnic [ 977.219573] uio [ 977.219573] cxgb4i [ 977.219573] cxgb4 [ 977.219574] cxgb3i [ 977.219574] cxgb3 [ 977.219574] mdio [ 977.219575] libcxgbi [ 977.219575] libcxgb [ 977.219575] qla4xxx [ 977.219575] iscsi_boot_sysfs [ 977.219576] mgag200 [ 977.219576] i2c_algo_bit [ 977.219576] 8021q [ 977.219576] drm_kms_helper [ 977.219577] garp [ 977.219577] syscopyarea [ 977.219577] mrp [ 977.219578] crct10dif_pclmul [ 977.219578] stp [ 977.219578] sysfillrect [ 977.219579] crct10dif_common [ 977.219579] sysimgblt [ 977.219579] crc32_pclmul [ 977.219579] llc [ 977.219580] fb_sys_fops [ 977.219580] crc32c_intel [ 977.219580] ttm [ 977.219581] ghash_clmulni_intel [ 977.219581] mlx5_core [ 977.219581] drm [ 977.219581] mxm_wmi [ 977.219582] ahci [ 977.219582] ixgbe(OE) [ 977.219582] aesni_intel [ 977.219583] mpt3sas [ 977.219583] mlxfw [ 977.219583] dm_multipath [ 977.219584] lrw [ 977.219584] devlink [ 977.219584] libahci [ 977.219585] gf128mul [ 977.219585] dca [ 977.219585] glue_helper [ 977.219585] ablk_helper [ 977.219586] raid_class [ 977.219586] ptp [ 977.219586] drm_panel_orientation_quirks [ 977.219587] libata [ 977.219587] cryptd [ 977.219587] scsi_transport_sas [ 977.219588] pps_core [ 977.219588] wmi [ 977.219588] sunrpc [ 977.219588] dm_mirror [ 977.219589] dm_region_hash [ 977.219589] dm_log [ 977.219589] dm_mod [ 977.219590] iscsi_tcp [ 977.219590] libiscsi_tcp [ 977.219590] libiscsi [ 977.219591] scsi_transport_iscsi [ 977.219591] fuse [ 977.219591] [ 977.219593] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.219594] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.219594] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 977.219595] RIP: 0010:[] [ 977.219597] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 977.219598] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 977.219598] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 977.219599] RDX: ffff97f2bfb5b8c0 RSI: 0000000000290000 RDI: ffff97f2a83bcf40 [ 977.219599] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 977.219600] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 977.219601] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 977.219601] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 977.219602] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.219603] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 977.219603] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.219604] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.219604] Call Trace: [ 977.219607] [] queued_spin_lock_slowpath+0xb/0xf [ 977.219608] [] _raw_spin_lock+0x30/0x40 [ 977.219614] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.219622] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.219624] [] ? wake_up_atomic_t+0x30/0x30 [ 977.219629] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.219650] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.219666] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 977.219668] [] ? enqueue_task_fair+0x208/0x6c0 [ 977.219669] [] ? check_preempt_curr+0x80/0xa0 [ 977.219670] [] ? ttwu_do_wakeup+0x19/0x100 [ 977.219694] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 977.219715] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.219717] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.219737] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.219755] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.219756] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.219757] [] ? sched_feat_set+0xf0/0xf0 [ 977.219776] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.219777] [] ? __switch_to+0xce/0x5a0 [ 977.219795] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.219797] [] kthread+0xd1/0xe0 [ 977.219798] [] ? insert_kthread_work+0x40/0x40 [ 977.219800] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.219801] [] ? insert_kthread_work+0x40/0x40 [ 977.219801] Code: [ 977.219802] c1 [ 977.219802] e8 [ 977.219802] 13 [ 977.219803] 48 [ 977.219803] c1 [ 977.219803] ea [ 977.219803] 0d [ 977.219804] 48 [ 977.219804] 98 [ 977.219804] 83 [ 977.219804] e2 [ 977.219805] 30 [ 977.219805] 48 [ 977.219805] 81 [ 977.219805] c2 [ 977.219806] c0 [ 977.219806] b8 [ 977.219806] 01 [ 977.219806] 00 [ 977.219807] 48 [ 977.219807] 03 [ 977.219807] 14 [ 977.219808] c5 [ 977.219808] e0 [ 977.219808] 17 [ 977.219808] d5 [ 977.219809] a0 [ 977.219809] 4c [ 977.219809] 89 [ 977.219809] 02 [ 977.219810] 41 [ 977.219810] 8b [ 977.219810] 40 [ 977.219810] 08 [ 977.219811] 85 [ 977.219811] c0 [ 977.219811] 75 [ 977.219812] 0f [ 977.219812] 0f [ 977.219812] 1f [ 977.219812] 44 [ 977.219813] 00 [ 977.219813] 00 [ 977.219813] [ 977.219813] 90 [ 977.219814] 41 [ 977.219814] 8b [ 977.219814] 40 [ 977.219814] 08 [ 977.219815] 85 [ 977.219815] c0 [ 977.219815] 74 [ 977.219815] f6 [ 977.219816] 4d [ 977.219816] 8b [ 977.219816] 08 [ 977.219816] 4d [ 977.219817] 85 [ 977.219817] c9 [ 977.219817] 74 [ 977.219817] 04 [ 977.219818] 41 [ 977.219818] 0f [ 977.219818] 18 [ 977.219818] [ 977.225548] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ll_mgs_0014:10826] [ 977.225548] Modules linked in: [ 977.225549] osp(OE) [ 977.225549] mdd(OE) [ 977.225550] lod(OE) [ 977.225550] mdt(OE) [ 977.225551] lfsck(OE) [ 977.225551] mgs(OE) [ 977.225551] mgc(OE) [ 977.225552] osd_zfs(OE) [ 977.225552] lquota(OE) [ 977.225552] fid(OE) [ 977.225553] fld(OE) [ 977.225553] ptlrpc(OE) [ 977.225553] obdclass(OE) [ 977.225553] ko2iblnd(OE) [ 977.225554] lnet(OE) [ 977.225554] libcfs(OE) [ 977.225554] iTCO_wdt [ 977.225555] iTCO_vendor_support [ 977.225555] sb_edac [ 977.225555] intel_powerclamp [ 977.225556] coretemp [ 977.225556] intel_rapl [ 977.225556] iosf_mbi [ 977.225556] rpcrdma [ 977.225557] ib_iser [ 977.225557] kvm [ 977.225557] irqbypass [ 977.225557] joydev [ 977.225558] pcspkr [ 977.225558] lpc_ich [ 977.225558] i2c_i801 [ 977.225559] ioatdma [ 977.225559] ses [ 977.225559] enclosure [ 977.225559] sg [ 977.225560] ipmi_si [ 977.225560] ipmi_devintf [ 977.225560] ipmi_msghandler [ 977.225561] acpi_power_meter [ 977.225561] dm_round_robin [ 977.225561] acpi_cpufreq [ 977.225562] ib_ipoib [ 977.225562] rdma_ucm [ 977.225562] ib_umad [ 977.225562] sch_fq_codel [ 977.225563] iw_cxgb4 [ 977.225563] rdma_cm [ 977.225563] iw_cm [ 977.225563] ib_cm [ 977.225564] iw_cxgb3 [ 977.225564] zfs(POE) [ 977.225564] binfmt_misc [ 977.225565] zunicode(POE) [ 977.225565] zavl(POE) [ 977.225565] icp(POE) [ 977.225566] zcommon(POE) [ 977.225566] znvpair(POE) [ 977.225566] spl(OE) [ 977.225567] msr_safe(OE) [ 977.225567] ip_tables [ 977.225567] nfsv3 [ 977.225567] nfs_acl [ 977.225568] rpcsec_gss_krb5 [ 977.225568] auth_rpcgss [ 977.225568] nfsv4 [ 977.225569] dns_resolver [ 977.225569] nfs [ 977.225569] lockd [ 977.225570] grace [ 977.225570] fscache [ 977.225570] overlay(T) [ 977.225571] ext4 [ 977.225571] mbcache [ 977.225571] jbd2 [ 977.225571] dm_service_time [ 977.225572] sd_mod [ 977.225572] crc_t10dif [ 977.225572] crct10dif_generic [ 977.225573] mlx5_ib [ 977.225573] ib_uverbs [ 977.225573] ib_core [ 977.225573] be2iscsi [ 977.225574] bnx2i [ 977.225574] cnic [ 977.225574] uio [ 977.225575] cxgb4i [ 977.225575] cxgb4 [ 977.225575] cxgb3i [ 977.225575] cxgb3 [ 977.225576] mdio [ 977.225576] libcxgbi [ 977.225576] libcxgb [ 977.225577] qla4xxx [ 977.225577] iscsi_boot_sysfs [ 977.225577] mgag200 [ 977.225577] i2c_algo_bit [ 977.225578] 8021q [ 977.225578] drm_kms_helper [ 977.225578] garp [ 977.225579] syscopyarea [ 977.225579] mrp [ 977.225579] crct10dif_pclmul [ 977.225580] stp [ 977.225580] sysfillrect [ 977.225580] crct10dif_common [ 977.225580] sysimgblt [ 977.225581] crc32_pclmul [ 977.225581] llc [ 977.225581] fb_sys_fops [ 977.225582] crc32c_intel [ 977.225582] ttm [ 977.225582] ghash_clmulni_intel [ 977.225582] mlx5_core [ 977.225583] drm [ 977.225583] mxm_wmi [ 977.225583] ahci [ 977.225583] ixgbe(OE) [ 977.225584] aesni_intel [ 977.225584] mpt3sas [ 977.225584] mlxfw [ 977.225585] dm_multipath [ 977.225585] lrw [ 977.225585] devlink [ 977.225585] libahci [ 977.225586] gf128mul [ 977.225586] dca [ 977.225586] glue_helper [ 977.225587] ablk_helper [ 977.225587] raid_class [ 977.225587] ptp [ 977.225588] drm_panel_orientation_quirks [ 977.225588] libata [ 977.225588] cryptd [ 977.225588] scsi_transport_sas [ 977.225589] pps_core [ 977.225589] wmi [ 977.225589] sunrpc [ 977.225590] dm_mirror [ 977.225590] dm_region_hash [ 977.225590] dm_log [ 977.225590] dm_mod [ 977.225591] iscsi_tcp [ 977.225591] libiscsi_tcp [ 977.225591] libiscsi [ 977.225592] scsi_transport_iscsi [ 977.225592] fuse [ 977.225592] [ 977.225594] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.225594] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.225595] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 977.225596] RIP: 0010:[] [ 977.225598] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 977.225598] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 977.225599] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 977.225600] RDX: ffff9832bf59b8c0 RSI: 0000000000f10000 RDI: ffff97f2a83bcf40 [ 977.225600] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 977.225601] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 977.225601] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 977.225602] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 977.225603] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.225604] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 977.225604] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.225605] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.225605] Call Trace: [ 977.225608] [] queued_spin_lock_slowpath+0xb/0xf [ 977.225609] [] _raw_spin_lock+0x30/0x40 [ 977.225615] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.225621] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.225623] [] ? wake_up_atomic_t+0x30/0x30 [ 977.225629] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.225647] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.225665] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 977.225666] [] ? check_preempt_curr+0x80/0xa0 [ 977.225684] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 977.225700] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 977.225723] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 977.225745] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.225746] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.225765] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.225784] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.225785] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.225786] [] ? sched_feat_set+0xf0/0xf0 [ 977.225804] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.225805] [] ? __switch_to+0xce/0x5a0 [ 977.225823] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.225825] [] kthread+0xd1/0xe0 [ 977.225826] [] ? insert_kthread_work+0x40/0x40 [ 977.225827] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.225829] [] ? insert_kthread_work+0x40/0x40 [ 977.225829] Code: [ 977.225830] 0d [ 977.225830] 48 [ 977.225830] 98 [ 977.225830] 83 [ 977.225831] e2 [ 977.225831] 30 [ 977.225831] 48 [ 977.225831] 81 [ 977.225832] c2 [ 977.225832] c0 [ 977.225832] b8 [ 977.225832] 01 [ 977.225833] 00 [ 977.225833] 48 [ 977.225833] 03 [ 977.225833] 14 [ 977.225834] c5 [ 977.225834] e0 [ 977.225834] 17 [ 977.225834] d5 [ 977.225835] a0 [ 977.225835] 4c [ 977.225835] 89 [ 977.225835] 02 [ 977.225836] 41 [ 977.225836] 8b [ 977.225836] 40 [ 977.225837] 08 [ 977.225837] 85 [ 977.225837] c0 [ 977.225837] 75 [ 977.225838] 0f [ 977.225838] 0f [ 977.225838] 1f [ 977.225838] 44 [ 977.225839] 00 [ 977.225839] 00 [ 977.225839] f3 [ 977.225839] 90 [ 977.225840] 41 [ 977.225840] 8b [ 977.225840] 40 [ 977.225840] 08 [ 977.225841] <85> [ 977.225841] c0 [ 977.225841] 74 [ 977.225842] f6 [ 977.225842] 4d [ 977.225842] 8b [ 977.225843] 08 [ 977.225843] 4d [ 977.225843] 85 [ 977.225843] c9 [ 977.225844] 74 [ 977.225844] 04 [ 977.225844] 41 [ 977.225844] 0f [ 977.225845] 18 [ 977.225845] 09 [ 977.225845] 8b [ 977.225845] 17 [ 977.225846] 0f [ 977.225846] b7 [ 977.225846] c2 [ 977.225846] [ 977.237549] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 23s! [mdt01_014:17318] [ 977.237550] Modules linked in: [ 977.237550] osp(OE) [ 977.237551] mdd(OE) [ 977.237551] lod(OE) [ 977.237552] mdt(OE) [ 977.237552] lfsck(OE) [ 977.237552] mgs(OE) [ 977.237553] mgc(OE) [ 977.237553] osd_zfs(OE) [ 977.237553] lquota(OE) [ 977.237554] fid(OE) [ 977.237554] fld(OE) [ 977.237554] ptlrpc(OE) [ 977.237555] obdclass(OE) [ 977.237555] ko2iblnd(OE) [ 977.237555] lnet(OE) [ 977.237555] libcfs(OE) [ 977.237556] iTCO_wdt [ 977.237556] iTCO_vendor_support [ 977.237556] sb_edac [ 977.237557] intel_powerclamp [ 977.237557] coretemp [ 977.237557] intel_rapl [ 977.237557] iosf_mbi [ 977.237558] rpcrdma [ 977.237558] ib_iser [ 977.237558] kvm [ 977.237559] irqbypass [ 977.237559] joydev [ 977.237559] pcspkr [ 977.237559] lpc_ich [ 977.237560] i2c_i801 [ 977.237560] ioatdma [ 977.237560] ses [ 977.237561] enclosure [ 977.237561] sg [ 977.237561] ipmi_si [ 977.237561] ipmi_devintf [ 977.237562] ipmi_msghandler [ 977.237562] acpi_power_meter [ 977.237562] dm_round_robin [ 977.237563] acpi_cpufreq [ 977.237563] ib_ipoib [ 977.237563] rdma_ucm [ 977.237563] ib_umad [ 977.237564] sch_fq_codel [ 977.237564] iw_cxgb4 [ 977.237564] rdma_cm [ 977.237565] iw_cm [ 977.237565] ib_cm [ 977.237565] iw_cxgb3 [ 977.237566] zfs(POE) [ 977.237566] binfmt_misc [ 977.237566] zunicode(POE) [ 977.237567] zavl(POE) [ 977.237567] icp(POE) [ 977.237567] zcommon(POE) [ 977.237568] znvpair(POE) [ 977.237568] spl(OE) [ 977.237568] msr_safe(OE) [ 977.237569] ip_tables [ 977.237569] nfsv3 [ 977.237569] nfs_acl [ 977.237569] rpcsec_gss_krb5 [ 977.237570] auth_rpcgss [ 977.237570] nfsv4 [ 977.237570] dns_resolver [ 977.237571] nfs [ 977.237571] lockd [ 977.237571] grace [ 977.237571] fscache [ 977.237572] overlay(T) [ 977.237572] ext4 [ 977.237572] mbcache [ 977.237573] jbd2 [ 977.237573] dm_service_time [ 977.237573] sd_mod [ 977.237574] crc_t10dif [ 977.237574] crct10dif_generic [ 977.237574] mlx5_ib [ 977.237575] ib_uverbs [ 977.237575] ib_core [ 977.237575] be2iscsi [ 977.237576] bnx2i [ 977.237576] cnic [ 977.237576] uio [ 977.237576] cxgb4i [ 977.237577] cxgb4 [ 977.237577] cxgb3i [ 977.237577] cxgb3 [ 977.237578] mdio [ 977.237578] libcxgbi [ 977.237578] libcxgb [ 977.237578] qla4xxx [ 977.237579] iscsi_boot_sysfs [ 977.237579] mgag200 [ 977.237579] i2c_algo_bit [ 977.237579] 8021q [ 977.237580] drm_kms_helper [ 977.237580] garp [ 977.237581] syscopyarea [ 977.237581] mrp [ 977.237581] crct10dif_pclmul [ 977.237581] stp [ 977.237582] sysfillrect [ 977.237582] crct10dif_common [ 977.237582] sysimgblt [ 977.237583] crc32_pclmul [ 977.237583] llc [ 977.237583] fb_sys_fops [ 977.237583] crc32c_intel [ 977.237584] ttm [ 977.237584] ghash_clmulni_intel [ 977.237584] mlx5_core [ 977.237585] drm [ 977.237585] mxm_wmi [ 977.237585] ahci [ 977.237586] ixgbe(OE) [ 977.237586] aesni_intel [ 977.237586] mpt3sas [ 977.237587] mlxfw [ 977.237587] dm_multipath [ 977.237587] lrw [ 977.237588] devlink [ 977.237588] libahci [ 977.237588] gf128mul [ 977.237588] dca [ 977.237589] glue_helper [ 977.237589] ablk_helper [ 977.237589] raid_class [ 977.237589] ptp [ 977.237590] drm_panel_orientation_quirks [ 977.237590] libata [ 977.237590] cryptd [ 977.237591] scsi_transport_sas [ 977.237591] pps_core [ 977.237591] wmi [ 977.237591] sunrpc [ 977.237592] dm_mirror [ 977.237592] dm_region_hash [ 977.237592] dm_log [ 977.237593] dm_mod [ 977.237593] iscsi_tcp [ 977.237593] libiscsi_tcp [ 977.237594] libiscsi [ 977.237594] scsi_transport_iscsi [ 977.237594] fuse [ 977.237595] [ 977.237596] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.237597] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.237598] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 977.237598] RIP: 0010:[] [ 977.237600] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 977.237601] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 977.237602] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 977.237603] RDX: ffff97f2bfc1b8c0 RSI: 0000000000810000 RDI: ffff97f2a83bcf40 [ 977.237603] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 977.237604] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 977.237605] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 977.237606] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 977.237606] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.237607] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 977.237608] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.237609] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.237609] Call Trace: [ 977.237611] [] queued_spin_lock_slowpath+0xb/0xf [ 977.237613] [] _raw_spin_lock+0x30/0x40 [ 977.237618] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.237625] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.237627] [] ? wake_up_atomic_t+0x30/0x30 [ 977.237632] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.237651] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.237669] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 977.237688] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 977.237705] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 977.237722] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 977.237745] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 977.237767] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.237768] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.237788] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.237807] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.237808] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.237809] [] ? sched_feat_set+0xf0/0xf0 [ 977.237827] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.237846] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.237847] [] kthread+0xd1/0xe0 [ 977.237849] [] ? insert_kthread_work+0x40/0x40 [ 977.237850] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.237851] [] ? insert_kthread_work+0x40/0x40 [ 977.237852] Code: [ 977.237852] 0d [ 977.237853] 48 [ 977.237853] 98 [ 977.237853] 83 [ 977.237853] e2 [ 977.237854] 30 [ 977.237854] 48 [ 977.237854] 81 [ 977.237854] c2 [ 977.237855] c0 [ 977.237855] b8 [ 977.237855] 01 [ 977.237855] 00 [ 977.237856] 48 [ 977.237856] 03 [ 977.237856] 14 [ 977.237857] c5 [ 977.237857] e0 [ 977.237857] 17 [ 977.237857] d5 [ 977.237858] a0 [ 977.237858] 4c [ 977.237858] 89 [ 977.237858] 02 [ 977.237859] 41 [ 977.237859] 8b [ 977.237859] 40 [ 977.237859] 08 [ 977.237860] 85 [ 977.237860] c0 [ 977.237861] 75 [ 977.237861] 0f [ 977.237861] 0f [ 977.237861] 1f [ 977.237862] 44 [ 977.237862] 00 [ 977.237862] 00 [ 977.237862] f3 [ 977.237863] 90 [ 977.237863] 41 [ 977.237863] 8b [ 977.237863] 40 [ 977.237864] 08 [ 977.237864] <85> [ 977.237864] c0 [ 977.237865] 74 [ 977.237865] f6 [ 977.237865] 4d [ 977.237865] 8b [ 977.237866] 08 [ 977.237866] 4d [ 977.237866] 85 [ 977.237866] c9 [ 977.237866] 74 [ 977.237867] 04 [ 977.237867] 41 [ 977.237867] 0f [ 977.237867] 18 [ 977.237868] 09 [ 977.237868] 8b [ 977.237868] 17 [ 977.237868] 0f [ 977.237869] b7 [ 977.237869] c2 [ 977.237869] [ 977.242552] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [mdt01_017:17555] [ 977.242552] Modules linked in: [ 977.242553] osp(OE) [ 977.242553] mdd(OE) [ 977.242554] lod(OE) [ 977.242554] mdt(OE) [ 977.242554] lfsck(OE) [ 977.242555] mgs(OE) [ 977.242555] mgc(OE) [ 977.242556] osd_zfs(OE) [ 977.242556] lquota(OE) [ 977.242556] fid(OE) [ 977.242557] fld(OE) [ 977.242557] ptlrpc(OE) [ 977.242558] obdclass(OE) [ 977.242558] ko2iblnd(OE) [ 977.242559] lnet(OE) [ 977.242559] libcfs(OE) [ 977.242560] iTCO_wdt [ 977.242560] iTCO_vendor_support [ 977.242560] sb_edac [ 977.242561] intel_powerclamp [ 977.242561] coretemp [ 977.242562] intel_rapl [ 977.242562] iosf_mbi [ 977.242562] rpcrdma [ 977.242563] ib_iser [ 977.242563] kvm [ 977.242563] irqbypass [ 977.242564] joydev [ 977.242564] pcspkr [ 977.242565] lpc_ich [ 977.242565] i2c_i801 [ 977.242565] ioatdma [ 977.242566] ses [ 977.242566] enclosure [ 977.242566] sg [ 977.242567] ipmi_si [ 977.242567] ipmi_devintf [ 977.242568] ipmi_msghandler [ 977.242568] acpi_power_meter [ 977.242568] dm_round_robin [ 977.242569] acpi_cpufreq [ 977.242569] ib_ipoib [ 977.242569] rdma_ucm [ 977.242570] ib_umad [ 977.242570] sch_fq_codel [ 977.242571] iw_cxgb4 [ 977.242571] rdma_cm [ 977.242571] iw_cm [ 977.242572] ib_cm [ 977.242572] iw_cxgb3 [ 977.242573] zfs(POE) [ 977.242573] binfmt_misc [ 977.242574] zunicode(POE) [ 977.242574] zavl(POE) [ 977.242575] icp(POE) [ 977.242575] zcommon(POE) [ 977.242576] znvpair(POE) [ 977.242576] spl(OE) [ 977.242577] msr_safe(OE) [ 977.242577] ip_tables [ 977.242577] nfsv3 [ 977.242578] nfs_acl [ 977.242578] rpcsec_gss_krb5 [ 977.242579] auth_rpcgss [ 977.242579] nfsv4 [ 977.242579] dns_resolver [ 977.242580] nfs [ 977.242580] lockd [ 977.242581] grace [ 977.242581] fscache [ 977.242582] overlay(T) [ 977.242582] ext4 [ 977.242582] mbcache [ 977.242583] jbd2 [ 977.242583] dm_service_time [ 977.242584] sd_mod [ 977.242584] crc_t10dif [ 977.242584] crct10dif_generic [ 977.242585] mlx5_ib [ 977.242585] ib_uverbs [ 977.242585] ib_core [ 977.242586] be2iscsi [ 977.242586] bnx2i [ 977.242587] cnic [ 977.242587] uio [ 977.242587] cxgb4i [ 977.242588] cxgb4 [ 977.242588] cxgb3i [ 977.242588] cxgb3 [ 977.242589] mdio [ 977.242589] libcxgbi [ 977.242590] libcxgb [ 977.242590] qla4xxx [ 977.242590] iscsi_boot_sysfs [ 977.242591] mgag200 [ 977.242591] i2c_algo_bit [ 977.242592] 8021q [ 977.242592] drm_kms_helper [ 977.242592] garp [ 977.242593] syscopyarea [ 977.242593] mrp [ 977.242594] crct10dif_pclmul [ 977.242594] stp [ 977.242594] sysfillrect [ 977.242595] crct10dif_common [ 977.242595] sysimgblt [ 977.242596] crc32_pclmul [ 977.242596] llc [ 977.242597] fb_sys_fops [ 977.242597] crc32c_intel [ 977.242597] ttm [ 977.242598] ghash_clmulni_intel [ 977.242598] mlx5_core [ 977.242599] drm [ 977.242599] mxm_wmi [ 977.242599] ahci [ 977.242600] ixgbe(OE) [ 977.242600] aesni_intel [ 977.242601] mpt3sas [ 977.242601] mlxfw [ 977.242601] dm_multipath [ 977.242602] lrw [ 977.242602] devlink [ 977.242602] libahci [ 977.242603] gf128mul [ 977.242603] dca [ 977.242604] glue_helper [ 977.242604] ablk_helper [ 977.242605] raid_class [ 977.242605] ptp [ 977.242606] drm_panel_orientation_quirks [ 977.242606] libata [ 977.242606] cryptd [ 977.242607] scsi_transport_sas [ 977.242607] pps_core [ 977.242608] wmi [ 977.242608] sunrpc [ 977.242608] dm_mirror [ 977.242609] dm_region_hash [ 977.242609] dm_log [ 977.242610] dm_mod [ 977.242610] iscsi_tcp [ 977.242610] libiscsi_tcp [ 977.242611] libiscsi [ 977.242611] scsi_transport_iscsi [ 977.242612] fuse [ 977.242612] [ 977.242614] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.242615] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.242616] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 977.242617] RIP: 0010:[] [ 977.242619] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 977.242620] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000246 [ 977.242621] RAX: 0000000000000000 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 977.242622] RDX: ffff9832bf35b8c0 RSI: 0000000000690001 RDI: ffff97f2a83bcf40 [ 977.242623] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: 0000000000000000 [ 977.242623] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 977.242624] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 977.242626] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 977.242627] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.242628] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 977.242628] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.242629] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.242630] Call Trace: [ 977.242632] [] queued_spin_lock_slowpath+0xb/0xf [ 977.242634] [] _raw_spin_lock+0x30/0x40 [ 977.242640] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.242648] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.242650] [] ? wake_up_atomic_t+0x30/0x30 [ 977.242658] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.242682] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.242707] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 977.242709] [] ? check_preempt_curr+0x80/0xa0 [ 977.242732] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 977.242755] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 977.242785] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 977.242813] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.242814] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.242840] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.242865] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.242867] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.242928] iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 977.242930] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.242931] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.242931] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 977.242934] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 977.242935] RSP: 0018:ffff98321b877a50 EFLAGS: 00000246 [ 977.242935] RAX: 0000000000000000 RBX: ffff97f2bfa80000 RCX: 0000000000110000 [ 977.242936] RDX: ffff9832bf4db8c0 RSI: 0000000000d90000 RDI: ffff97f2a83bcf40 [ 977.242936] RBP: ffff98321b877a50 R08: ffff97f2bfa9b8c0 R09: 0000000000000000 [ 977.242937] R10: 0000000000000001 R11: 000000000000242e R12: ffff98321b877a18 [ 977.242937] R13: ffff97f2bfc9ad40 R14: ffffffffa00f0602 R15: ffff98321b877a28 [ 977.242938] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 977.242939] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.242940] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 977.242940] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.242941] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.242941] Call Trace: [ 977.242943] [] queued_spin_lock_slowpath+0xb/0xf [ 977.242945] [] _raw_spin_lock+0x30/0x40 [ 977.242950] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.242957] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.242959] [] ? wake_up_atomic_t+0x30/0x30 [ 977.242964] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.242983] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.242999] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 977.243017] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 977.243018] [] ? enqueue_task_fair+0x208/0x6c0 [ 977.243020] [] ? sched_clock_cpu+0xa5/0xe0 [ 977.243021] [] ? check_preempt_curr+0x80/0xa0 [ 977.243022] [] ? ttwu_do_wakeup+0x19/0x100 [ 977.243045] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 977.243066] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.243068] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.243087] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.243107] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.243109] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.243110] [] ? sched_feat_set+0xf0/0xf0 [ 977.243128] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.243130] [] ? __switch_to+0xce/0x5a0 [ 977.243148] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.243149] [] kthread+0xd1/0xe0 [ 977.243151] [] ? insert_kthread_work+0x40/0x40 [ 977.243152] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.243153] [] ? insert_kthread_work+0x40/0x40 [ 977.243167] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 977.244552] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [mdt01_018:17592] [ 977.244580] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 977.244601] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 977.244603] CPU: 30 PID: 17592 Comm: mdt01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 977.244603] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 977.244604] task: ffff983230579080 ti: ffff9832469b0000 task.ti: ffff9832469b0000 [ 977.244607] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 977.244608] RSP: 0018:ffff9832469b3a50 EFLAGS: 00000246 [ 977.244609] RAX: 0000000000000000 RBX: ffff9832bf580000 RCX: 0000000000f10000 [ 977.244610] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10001 RDI: ffff97f2a83bcf40 [ 977.244611] RBP: ffff9832469b3a50 R08: ffff9832bf59b8c0 R09: 0000000000000000 [ 977.244612] R10: 0000000000000002 R11: ffff983211336600 R12: ffff9832469b3a18 [ 977.244613] R13: ffffffffa00ec9b1 R14: ffff9832469b39f8 R15: ffff9832469b3a28 [ 977.244614] FS: 0000000000000000(0000) GS:ffff9832bf580000(0000) knlGS:0000000000000000 [ 977.244615] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.244616] CR2: 00007ffff7fc5010 CR3: 0000003f74514000 CR4: 00000000003607e0 [ 977.244617] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 977.244617] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 977.244618] Call Trace: [ 977.244620] [] queued_spin_lock_slowpath+0xb/0xf [ 977.244621] [] _raw_spin_lock+0x30/0x40 [ 977.244628] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 977.244636] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 977.244638] [] ? wake_up_atomic_t+0x30/0x30 [ 977.244646] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 977.244671] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 977.244695] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 977.244697] [] ? check_preempt_curr+0x80/0xa0 [ 977.244698] [] ? ttwu_do_wakeup+0x19/0x100 [ 977.244729] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 977.244757] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 977.244758] [] ? __getnstimeofday64+0x3f/0xd0 [ 977.244784] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 977.244808] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 977.244810] [] ? __wake_up_common_lock+0x91/0xc0 [ 977.244811] [] ? sched_feat_set+0xf0/0xf0 [ 977.244836] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 977.244861] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 977.244863] [] kthread+0xd1/0xe0 [ 977.244865] [] ? insert_kthread_work+0x40/0x40 [ 977.244867] [] ret_from_fork_nospec_begin+0x21/0x21 [ 977.244869] [] ? insert_kthread_work+0x40/0x40 [ 977.244888] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 978.597729] Lustre: ls1-MDT0007-osp-MDT0000: Connection to ls1-MDT0007 (at 172.19.3.105@o2ib600) was lost; in progress operations using this service will wait for recovery to complete [ 978.597730] Lustre: Skipped 3 previous similar messages [ 981.525553] [] ? sched_feat_set+0xf0/0xf0 [ 981.532380] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 981.541404] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 981.551673] [] kthread+0xd1/0xe0 [ 981.557603] [] ? insert_kthread_work+0x40/0x40 [ 981.566434] [] ret_from_fork_nospec_begin+0x21/0x21 [ 981.574207] [] ? insert_kthread_work+0x40/0x40 [ 981.581491] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 981.607063] Pid: 17321, comm: mdt00_009 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 SMP Tue Jan 25 12:06:24 PST 2022 [ 981.620157] Call Trace: [ 981.623376] [] 0xffffffffffffffff [ 981.632513] LustreError: dumping log to /tmp/lustre-log.1644342637.17321 [ 984.604453] LustreError: 15424:0:(osp_precreate.c:964:osp_precreate_cleanup_orphans()) ls1-OST0018-osc-MDT0000: cannot cleanup orphans: rc = -11 [ 989.210929] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [mdt00_031:18467] [ 989.214929] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [mdt00_030:16183] [ 989.214967] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 989.214994] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 989.214996] CPU: 17 PID: 16183 Comm: mdt00_030 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 989.214996] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 989.214997] task: ffff97f250d53180 ti: ffff97f250d78000 task.ti: ffff97f250d78000 [ 989.215003] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 989.215003] RSP: 0018:ffff97f250d7bab0 EFLAGS: 00000246 [ 989.215004] RAX: 0000000000000000 RBX: ffff97f2bfc40000 RCX: 0000000000890000 [ 989.215005] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 989.215005] RBP: ffff97f250d7bab0 R08: ffff97f2bfc5b8c0 R09: 0000000000000000 [ 989.215006] R10: 0000000000000001 R11: 0000000000000400 R12: ffff97f250d7ba78 [ 989.215006] R13: ffffffffa00ec9b1 R14: ffff97f250d7ba58 R15: ffff97f250d7ba88 [ 989.215007] FS: 0000000000000000(0000) GS:ffff97f2bfc40000(0000) knlGS:0000000000000000 [ 989.215008] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 989.215009] CR2: 00007ffff75f34b1 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 989.215010] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 989.215010] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 989.215010] Call Trace: [ 989.215015] [] queued_spin_lock_slowpath+0xb/0xf [ 989.215018] [] _raw_spin_lock+0x30/0x40 [ 989.215032] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 989.215036] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 989.215049] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 989.215052] [] ? wake_up_atomic_t+0x30/0x30 [ 989.215058] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 989.215105] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 989.215123] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 989.215126] [] ? check_preempt_curr+0x80/0xa0 [ 989.215143] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 989.215160] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 989.215192] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 989.215213] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 989.215215] [] ? __getnstimeofday64+0x3f/0xd0 [ 989.215236] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 989.215254] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 989.215255] [] ? __wake_up_common_lock+0x91/0xc0 [ 989.215257] [] ? sched_feat_set+0xf0/0xf0 [ 989.215275] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 989.215278] [] ? __switch_to+0xce/0x5a0 [ 989.215296] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 989.215298] [] kthread+0xd1/0xe0 [ 989.215299] [] ? insert_kthread_work+0x40/0x40 [ 989.215301] [] ret_from_fork_nospec_begin+0x21/0x21 [ 989.215302] [] ? insert_kthread_work+0x40/0x40 [ 989.215316] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 989.728740] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 989.808581] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 989.869663] CPU: 16 PID: 18467 Comm: mdt00_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 989.884026] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 989.896353] task: ffff97f249f7a100 ti: ffff97f249078000 task.ti: ffff97f249078000 [ 989.905188] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 989.916752] RSP: 0018:ffff97f24907bab0 EFLAGS: 00000246 [ 989.923164] RAX: 0000000000000000 RBX: ffff97f2bfc00000 RCX: 0000000000810000 [ 989.931613] RDX: ffff97f2bfddb8c0 RSI: 0000000000b90000 RDI: ffff97f2a83bcf40 [ 989.940059] RBP: ffff97f24907bab0 R08: ffff97f2bfc1b8c0 R09: 0000000000000000 [ 989.948508] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f24907ba78 [ 989.956955] R13: ffffffffa00ec9b1 R14: ffff97f24907ba58 R15: ffff97f24907ba88 [ 989.965401] FS: 0000000000000000(0000) GS:ffff97f2bfc00000(0000) knlGS:0000000000000000 [ 989.974917] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 989.981814] CR2: 00007ffff7ad2d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 989.990261] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 989.998710] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 990.007158] Call Trace: [ 990.010371] [] queued_spin_lock_slowpath+0xb/0xf [ 990.017849] [] _raw_spin_lock+0x30/0x40 [ 990.024459] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 990.032138] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 990.040587] [] ? wake_up_atomic_t+0x30/0x30 [ 990.047586] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 990.054991] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 990.063269] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 990.071257] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 990.079922] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 990.087810] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 990.095123] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 990.103308] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 990.112240] [] ? __getnstimeofday64+0x3f/0xd0 [ 990.119451] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 990.128506] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 990.136566] [] ? __wake_up_common_lock+0x91/0xc0 [ 990.144044] [] ? sched_feat_set+0xf0/0xf0 [ 990.150868] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 990.158348] [] ? __switch_to+0xce/0x5a0 [ 990.164978] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 990.173718] [] kthread+0xd1/0xe0 [ 990.179647] [] ? insert_kthread_work+0x40/0x40 [ 990.186931] [] ret_from_fork_nospec_begin+0x21/0x21 [ 990.194702] [] ? insert_kthread_work+0x40/0x40 [ 990.201985] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 993.069374] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [mdt00_008:17320] [ 993.083374] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_mgs_0009:8655] [ 993.078404] Modules linked in: osp(OE) mdd(OE) [ 993.083375] Modules linked in: [ 993.083375] osp(OE) [ 993.083376] mdd(OE) [ 993.083376] lod(OE) [ 993.083377] mdt(OE) [ 993.083377] lfsck(OE) [ 993.083377] mgs(OE) [ 993.083378] mgc(OE) [ 993.083378] osd_zfs(OE) [ 993.083378] lquota(OE) [ 993.083379] fid(OE) [ 993.083379] fld(OE) [ 993.083379] ptlrpc(OE) [ 993.083380] obdclass(OE) [ 993.083380] ko2iblnd(OE) [ 993.083380] lnet(OE) [ 993.083381] libcfs(OE) [ 993.083381] iTCO_wdt [ 993.083381] iTCO_vendor_support [ 993.083382] sb_edac [ 993.083382] intel_powerclamp [ 993.083382] coretemp [ 993.083382] intel_rapl [ 993.083383] iosf_mbi [ 993.083383] rpcrdma [ 993.083383] ib_iser [ 993.083384] kvm [ 993.083384] irqbypass [ 993.083384] joydev [ 993.083384] pcspkr [ 993.083385] lpc_ich [ 993.083385] i2c_i801 [ 993.083385] ioatdma [ 993.083386] ses [ 993.083386] enclosure [ 993.083386] sg [ 993.083386] ipmi_si [ 993.083387] ipmi_devintf [ 993.083387] ipmi_msghandler [ 993.083387] acpi_power_meter [ 993.083387] dm_round_robin [ 993.083388] acpi_cpufreq [ 993.083388] ib_ipoib [ 993.083388] rdma_ucm [ 993.083389] ib_umad [ 993.083389] sch_fq_codel [ 993.083389] iw_cxgb4 [ 993.083389] rdma_cm [ 993.083390] iw_cm [ 993.083390] ib_cm [ 993.083390] iw_cxgb3 [ 993.083391] zfs(POE) [ 993.083391] binfmt_misc [ 993.083391] zunicode(POE) [ 993.083392] zavl(POE) [ 993.083392] icp(POE) [ 993.083392] zcommon(POE) [ 993.083392] znvpair(POE) [ 993.083393] spl(OE) [ 993.083393] msr_safe(OE) [ 993.083393] ip_tables [ 993.083394] nfsv3 [ 993.083394] nfs_acl [ 993.083394] rpcsec_gss_krb5 [ 993.083395] auth_rpcgss [ 993.083395] nfsv4 [ 993.083395] dns_resolver [ 993.083395] nfs [ 993.083396] lockd [ 993.083396] grace [ 993.083396] fscache [ 993.083397] overlay(T) [ 993.083397] ext4 [ 993.083397] mbcache [ 993.083398] jbd2 [ 993.083398] dm_service_time [ 993.083398] sd_mod [ 993.083399] crc_t10dif [ 993.083399] crct10dif_generic [ 993.083399] mlx5_ib [ 993.083400] ib_uverbs [ 993.083400] ib_core [ 993.083400] be2iscsi [ 993.083401] bnx2i [ 993.083401] cnic [ 993.083401] uio [ 993.083401] cxgb4i [ 993.083402] cxgb4 [ 993.083402] cxgb3i [ 993.083402] cxgb3 [ 993.083403] mdio [ 993.083403] libcxgbi [ 993.083403] libcxgb [ 993.083404] qla4xxx [ 993.083404] iscsi_boot_sysfs [ 993.083404] mgag200 [ 993.083404] i2c_algo_bit [ 993.083405] 8021q [ 993.083405] drm_kms_helper [ 993.083405] garp [ 993.083406] syscopyarea [ 993.083406] mrp [ 993.083406] crct10dif_pclmul [ 993.083406] stp [ 993.083407] sysfillrect [ 993.083407] crct10dif_common [ 993.083407] sysimgblt [ 993.083408] crc32_pclmul [ 993.083408] llc [ 993.083408] fb_sys_fops [ 993.083409] crc32c_intel [ 993.083409] ttm [ 993.083409] ghash_clmulni_intel [ 993.083410] mlx5_core [ 993.083410] drm [ 993.083410] mxm_wmi [ 993.083410] ahci [ 993.083411] ixgbe(OE) [ 993.083411] aesni_intel [ 993.083411] mpt3sas [ 993.083412] mlxfw [ 993.083412] dm_multipath [ 993.083412] lrw [ 993.083413] devlink [ 993.083413] libahci [ 993.083413] gf128mul [ 993.083413] dca [ 993.083414] glue_helper [ 993.083414] ablk_helper [ 993.083414] raid_class [ 993.083415] ptp [ 993.083415] drm_panel_orientation_quirks [ 993.083415] libata [ 993.083416] cryptd [ 993.083416] scsi_transport_sas [ 993.083416] pps_core [ 993.083417] wmi [ 993.083417] sunrpc [ 993.083417] dm_mirror [ 993.083417] dm_region_hash [ 993.083418] dm_log [ 993.083418] dm_mod [ 993.083418] iscsi_tcp [ 993.083419] libiscsi_tcp [ 993.083419] libiscsi [ 993.083419] scsi_transport_iscsi [ 993.083420] fuse [ 993.083420] [ 993.083421] CPU: 3 PID: 8655 Comm: ll_mgs_0009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 993.083422] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 993.083423] task: ffff983229036300 ti: ffff98322904c000 task.ti: ffff98322904c000 [ 993.083424] RIP: 0010:[] [ 993.083426] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 993.083426] RSP: 0018:ffff98322904fa50 EFLAGS: 00000246 [ 993.083427] RAX: 0000000000000000 RBX: ffff97f2bfac0000 RCX: 0000000000190000 [ 993.083427] RDX: ffff9832bf21b8c0 RSI: 0000000000410000 RDI: ffff97f2a83bcf40 [ 993.083428] RBP: ffff98322904fa50 R08: ffff97f2bfadb8c0 R09: 0000000000000000 [ 993.083429] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98322904fa18 [ 993.083429] R13: ffffffffa00ec9b1 R14: ffff98322904f9f8 R15: ffff98322904fa28 [ 993.083430] FS: 0000000000000000(0000) GS:ffff97f2bfac0000(0000) knlGS:0000000000000000 [ 993.083431] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 993.083431] CR2: 00007ffff579f624 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 993.083432] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 993.083433] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 993.083433] Call Trace: [ 993.083435] [] queued_spin_lock_slowpath+0xb/0xf [ 993.083437] [] _raw_spin_lock+0x30/0x40 [ 993.083443] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 993.083447] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 993.083454] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 993.083455] [] ? wake_up_atomic_t+0x30/0x30 [ 993.083461] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 993.083482] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 993.083499] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 993.083505] [] ? lnet_ptl_attach_md+0x38e/0x540 [lnet] [ 993.083508] [] ? enqueue_task_fair+0x208/0x6c0 [ 993.083509] [] ? sched_clock_cpu+0xa5/0xe0 [ 993.083510] [] ? check_preempt_curr+0x80/0xa0 [ 993.083512] [] ? ttwu_do_wakeup+0x19/0x100 [ 993.083535] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 993.083556] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 993.083557] [] ? __getnstimeofday64+0x3f/0xd0 [ 993.083577] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 993.083595] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 993.083597] [] ? __wake_up_common_lock+0x91/0xc0 [ 993.083598] [] ? sched_feat_set+0xf0/0xf0 [ 993.083616] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 993.083617] [] ? __switch_to+0xce/0x5a0 [ 993.083635] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 993.083637] [] kthread+0xd1/0xe0 [ 993.083638] [] ? insert_kthread_work+0x40/0x40 [ 993.083640] [] ret_from_fork_nospec_begin+0x21/0x21 [ 993.083641] [] ? insert_kthread_work+0x40/0x40 [ 993.083642] Code: [ 993.083642] c1 [ 993.083642] e8 [ 993.083642] 13 [ 993.083643] 48 [ 993.083643] c1 [ 993.083643] ea [ 993.083643] 0d [ 993.083644] 48 [ 993.083644] 98 [ 993.083644] 83 [ 993.083644] e2 [ 993.083645] 30 [ 993.083645] 48 [ 993.083645] 81 [ 993.083645] c2 [ 993.083646] c0 [ 993.083646] b8 [ 993.083646] 01 [ 993.083646] 00 [ 993.083647] 48 [ 993.083647] 03 [ 993.083647] 14 [ 993.083647] c5 [ 993.083648] e0 [ 993.083648] 17 [ 993.083648] d5 [ 993.083648] a0 [ 993.083649] 4c [ 993.083649] 89 [ 993.083649] 02 [ 993.083649] 41 [ 993.083650] 8b [ 993.083650] 40 [ 993.083650] 08 [ 993.083650] 85 [ 993.083651] c0 [ 993.083651] 75 [ 993.083651] 0f [ 993.083652] 0f [ 993.083652] 1f [ 993.083652] 44 [ 993.083652] 00 [ 993.083653] 00 [ 993.083653] [ 993.083653] 90 [ 993.083653] 41 [ 993.083654] 8b [ 993.083654] 40 [ 993.083654] 08 [ 993.083655] 85 [ 993.083655] c0 [ 993.083655] 74 [ 993.083655] f6 [ 993.083656] 4d [ 993.083656] 8b [ 993.083656] 08 [ 993.083656] 4d [ 993.083657] 85 [ 993.083657] c9 [ 993.083657] 74 [ 993.083657] 04 [ 993.083658] 41 [ 993.083658] 0f [ 993.083658] 18 [ 993.083658] [ 993.091375] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [mdt00_002:10663] [ 993.091376] Modules linked in: [ 993.091376] osp(OE) [ 993.091377] mdd(OE) [ 993.091377] lod(OE) [ 993.091378] mdt(OE) [ 993.091378] lfsck(OE) [ 993.091379] mgs(OE) [ 993.091379] mgc(OE) [ 993.091380] osd_zfs(OE) [ 993.091380] lquota(OE) [ 993.091381] fid(OE) [ 993.091381] fld(OE) [ 993.091382] ptlrpc(OE) [ 993.091382] obdclass(OE) [ 993.091383] ko2iblnd(OE) [ 993.091383] lnet(OE) [ 993.091384] libcfs(OE) [ 993.091384] iTCO_wdt [ 993.091384] iTCO_vendor_support [ 993.091385] sb_edac [ 993.091385] intel_powerclamp [ 993.091386] coretemp [ 993.091386] intel_rapl [ 993.091387] iosf_mbi [ 993.091387] rpcrdma [ 993.091387] ib_iser [ 993.091388] kvm [ 993.091388] irqbypass [ 993.091389] joydev [ 993.091389] pcspkr [ 993.091389] lpc_ich [ 993.091390] i2c_i801 [ 993.091390] ioatdma [ 993.091390] ses [ 993.091391] enclosure [ 993.091391] sg [ 993.091392] ipmi_si [ 993.091392] ipmi_devintf [ 993.091392] ipmi_msghandler [ 993.091393] acpi_power_meter [ 993.091393] dm_round_robin [ 993.091394] acpi_cpufreq [ 993.091394] ib_ipoib [ 993.091394] rdma_ucm [ 993.091395] ib_umad [ 993.091395] sch_fq_codel [ 993.091396] iw_cxgb4 [ 993.091396] rdma_cm [ 993.091397] iw_cm [ 993.091397] ib_cm [ 993.091397] iw_cxgb3 [ 993.091398] zfs(POE) [ 993.091398] binfmt_misc [ 993.091399] zunicode(POE) [ 993.091399] zavl(POE) [ 993.091400] icp(POE) [ 993.091400] zcommon(POE) [ 993.091401] znvpair(POE) [ 993.091401] spl(OE) [ 993.091402] msr_safe(OE) [ 993.091402] ip_tables [ 993.091403] nfsv3 [ 993.091403] nfs_acl [ 993.091403] rpcsec_gss_krb5 [ 993.091404] auth_rpcgss [ 993.091404] nfsv4 [ 993.091405] dns_resolver [ 993.091405] nfs [ 993.091405] lockd [ 993.091406] grace [ 993.091406] fscache [ 993.091407] overlay(T) [ 993.091407] ext4 [ 993.091408] mbcache [ 993.091408] jbd2 [ 993.091409] dm_service_time [ 993.091409] sd_mod [ 993.091409] crc_t10dif [ 993.091410] crct10dif_generic [ 993.091410] mlx5_ib [ 993.091411] ib_uverbs [ 993.091411] ib_core [ 993.091411] be2iscsi [ 993.091412] bnx2i [ 993.091413] cnic [ 993.091413] uio [ 993.091413] cxgb4i [ 993.091414] cxgb4 [ 993.091414] cxgb3i [ 993.091415] cxgb3 [ 993.091415] mdio [ 993.091415] libcxgbi [ 993.091416] libcxgb [ 993.091416] qla4xxx [ 993.091417] iscsi_boot_sysfs [ 993.091417] mgag200 [ 993.091417] i2c_algo_bit [ 993.091418] 8021q [ 993.091418] drm_kms_helper [ 993.091419] garp [ 993.091419] syscopyarea [ 993.091419] mrp [ 993.091420] crct10dif_pclmul [ 993.091420] stp [ 993.091421] sysfillrect [ 993.091421] crct10dif_common [ 993.091422] sysimgblt [ 993.091422] crc32_pclmul [ 993.091422] llc [ 993.091423] fb_sys_fops [ 993.091423] crc32c_intel [ 993.091424] ttm [ 993.091424] ghash_clmulni_intel [ 993.091424] mlx5_core [ 993.091425] drm [ 993.091425] mxm_wmi [ 993.091426] ahci [ 993.091426] ixgbe(OE) [ 993.091427] aesni_intel [ 993.091427] mpt3sas [ 993.091427] mlxfw [ 993.091428] dm_multipath [ 993.091428] lrw [ 993.091429] devlink [ 993.091429] libahci [ 993.091429] gf128mul [ 993.091430] dca [ 993.091430] glue_helper [ 993.091431] ablk_helper [ 993.091431] raid_class [ 993.091431] ptp [ 993.091432] drm_panel_orientation_quirks [ 993.091432] libata [ 993.091433] cryptd [ 993.091433] scsi_transport_sas [ 993.091433] pps_core [ 993.091434] wmi [ 993.091434] sunrpc [ 993.091435] dm_mirror [ 993.091435] dm_region_hash [ 993.091435] dm_log [ 993.091436] dm_mod [ 993.091436] iscsi_tcp [ 993.091437] libiscsi_tcp [ 993.091437] libiscsi [ 993.091437] scsi_transport_iscsi [ 993.091438] fuse [ 993.091438] [ 993.091440] CPU: 5 PID: 10663 Comm: mdt00_002 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 993.091441] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 993.091442] task: ffff97f28e72a100 ti: ffff97f28e600000 task.ti: ffff97f28e600000 [ 993.091443] RIP: 0010:[] [ 993.091445] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 993.091446] RSP: 0018:ffff97f28e603ab0 EFLAGS: 00000246 [ 993.091447] RAX: 0000000000000000 RBX: ffff97f2bfb40000 RCX: 0000000000290000 [ 993.091448] RDX: ffff97f2bfc1b8c0 RSI: 0000000000810000 RDI: ffff97f2a83bcf40 [ 993.091449] RBP: ffff97f28e603ab0 R08: ffff97f2bfb5b8c0 R09: 0000000000000000 [ 993.091450] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f28e603a78 [ 993.091451] R13: ffffffffa00ec9b1 R14: ffff97f28e603a58 R15: ffff97f28e603a88 [ 993.091452] FS: 0000000000000000(0000) GS:ffff97f2bfb40000(0000) knlGS:0000000000000000 [ 993.091453] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 993.091454] CR2: 00007ffff1096160 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 993.091455] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 993.091456] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 993.091456] Call Trace: [ 993.091459] [] queued_spin_lock_slowpath+0xb/0xf [ 993.091461] [] _raw_spin_lock+0x30/0x40 [ 993.091468] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 993.091474] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 993.091482] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 993.091484] [] ? wake_up_atomic_t+0x30/0x30 [ 993.091492] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 993.091516] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 993.091541] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 993.091566] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 993.091590] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 993.091613] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 993.091643] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 993.091671] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 993.091673] [] ? __getnstimeofday64+0x3f/0xd0 [ 993.091698] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 993.091722] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 993.091724] [] ? __wake_up_common_lock+0x91/0xc0 [ 993.091726] [] ? sched_feat_set+0xf0/0xf0 [ 993.091750] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 993.091752] [] ? __switch_to+0xce/0x5a0 [ 993.091776] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 993.091778] [] kthread+0xd1/0xe0 [ 993.091780] [] ? insert_kthread_work+0x40/0x40 [ 993.091782] [] ret_from_fork_nospec_begin+0x21/0x21 [ 993.091783] [] ? insert_kthread_work+0x40/0x40 [ 993.091784] Code: [ 993.091785] 0d [ 993.091785] 48 [ 993.091785] 98 [ 993.091786] 83 [ 993.091786] e2 [ 993.091786] 30 [ 993.091787] 48 [ 993.091787] 81 [ 993.091787] c2 [ 993.091788] c0 [ 993.091788] b8 [ 993.091789] 01 [ 993.091789] 00 [ 993.091789] 48 [ 993.091790] 03 [ 993.091790] 14 [ 993.091790] c5 [ 993.091791] e0 [ 993.091791] 17 [ 993.091791] d5 [ 993.091792] a0 [ 993.091792] 4c [ 993.091792] 89 [ 993.091793] 02 [ 993.091793] 41 [ 993.091794] 8b [ 993.091794] 40 [ 993.091794] 08 [ 993.091795] 85 [ 993.091795] c0 [ 993.091795] 75 [ 993.091796] 0f [ 993.091796] 0f [ 993.091797] 1f [ 993.091797] 44 [ 993.091797] 00 [ 993.091798] 00 [ 993.091798] f3 [ 993.091798] 90 [ 993.091799] 41 [ 993.091799] 8b [ 993.091799] 40 [ 993.091800] 08 [ 993.091800] <85> [ 993.091801] c0 [ 993.091801] 74 [ 993.091801] f6 [ 993.091802] 4d [ 993.091802] 8b [ 993.091802] 08 [ 993.091803] 4d [ 993.091803] 85 [ 993.091803] c9 [ 993.091804] 74 [ 993.091804] 04 [ 993.091804] 41 [ 993.091805] 0f [ 993.091805] 18 [ 993.091805] 09 [ 993.091806] 8b [ 993.091806] 17 [ 993.091806] 0f [ 993.091807] b7 [ 993.091807] c2 [ 993.091807] [ 993.198390] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [mdt01_003:10825] [ 993.198390] Modules linked in: [ 993.198391] osp(OE) [ 993.198392] mdd(OE) [ 993.198393] lod(OE) [ 993.198393] mdt(OE) [ 993.198394] lfsck(OE) [ 993.198394] mgs(OE) [ 993.198395] mgc(OE) [ 993.198395] osd_zfs(OE) [ 993.198396] lquota(OE) [ 993.198396] fid(OE) [ 993.198397] fld(OE) [ 993.198397] ptlrpc(OE) [ 993.198398] obdclass(OE) [ 993.198398] ko2iblnd(OE) [ 993.198399] lnet(OE) [ 993.198399] libcfs(OE) [ 993.198400] iTCO_wdt [ 993.198401] iTCO_vendor_support [ 993.198401] sb_edac [ 993.198402] intel_powerclamp [ 993.198402] coretemp [ 993.198403] intel_rapl [ 993.198403] iosf_mbi [ 993.198404] rpcrdma [ 993.198404] ib_iser [ 993.198405] kvm [ 993.198405] irqbypass [ 993.198406] joydev [ 993.198406] pcspkr [ 993.198407] lpc_ich [ 993.198407] i2c_i801 [ 993.198408] ioatdma [ 993.198408] ses [ 993.198409] enclosure [ 993.198409] sg [ 993.198410] ipmi_si [ 993.198410] ipmi_devintf [ 993.198411] ipmi_msghandler [ 993.198411] acpi_power_meter [ 993.198412] dm_round_robin [ 993.198413] acpi_cpufreq [ 993.198413] ib_ipoib [ 993.198414] rdma_ucm [ 993.198414] ib_umad [ 993.198415] sch_fq_codel [ 993.198415] iw_cxgb4 [ 993.198416] rdma_cm [ 993.198416] iw_cm [ 993.198416] ib_cm [ 993.198417] iw_cxgb3 [ 993.198418] zfs(POE) [ 993.198418] binfmt_misc [ 993.198419] zunicode(POE) [ 993.198420] zavl(POE) [ 993.198420] icp(POE) [ 993.198421] zcommon(POE) [ 993.198421] znvpair(POE) [ 993.198422] spl(OE) [ 993.198422] msr_safe(OE) [ 993.198423] ip_tables [ 993.198423] nfsv3 [ 993.198424] nfs_acl [ 993.198424] rpcsec_gss_krb5 [ 993.198425] auth_rpcgss [ 993.198425] nfsv4 [ 993.198426] dns_resolver [ 993.198426] nfs [ 993.198427] lockd [ 993.198427] grace [ 993.198428] fscache [ 993.198428] overlay(T) [ 993.198429] ext4 [ 993.198429] mbcache [ 993.198430] jbd2 [ 993.198430] dm_service_time [ 993.198431] sd_mod [ 993.198431] crc_t10dif [ 993.198432] crct10dif_generic [ 993.198432] mlx5_ib [ 993.198433] ib_uverbs [ 993.198433] ib_core [ 993.198433] be2iscsi [ 993.198434] bnx2i [ 993.198434] cnic [ 993.198435] uio [ 993.198435] cxgb4i [ 993.198436] cxgb4 [ 993.198436] cxgb3i [ 993.198437] cxgb3 [ 993.198437] mdio [ 993.198438] libcxgbi [ 993.198438] libcxgb [ 993.198439] qla4xxx [ 993.198439] iscsi_boot_sysfs [ 993.198439] mgag200 [ 993.198440] i2c_algo_bit [ 993.198441] 8021q [ 993.198441] drm_kms_helper [ 993.198442] garp [ 993.198442] syscopyarea [ 993.198443] mrp [ 993.198443] crct10dif_pclmul [ 993.198444] stp [ 993.198444] sysfillrect [ 993.198445] crct10dif_common [ 993.198445] sysimgblt [ 993.198446] crc32_pclmul [ 993.198446] llc [ 993.198447] fb_sys_fops [ 993.198447] crc32c_intel [ 993.198448] ttm [ 993.198448] ghash_clmulni_intel [ 993.198449] mlx5_core [ 993.198449] drm [ 993.198450] mxm_wmi [ 993.198450] ahci [ 993.198451] ixgbe(OE) [ 993.198451] aesni_intel [ 993.198452] mpt3sas [ 993.198452] mlxfw [ 993.198453] dm_multipath [ 993.198453] lrw [ 993.198454] devlink [ 993.198454] libahci [ 993.198454] gf128mul [ 993.198455] dca [ 993.198455] glue_helper [ 993.198456] ablk_helper [ 993.198456] raid_class [ 993.198457] ptp [ 993.198457] drm_panel_orientation_quirks [ 993.198458] libata [ 993.198458] cryptd [ 993.198459] scsi_transport_sas [ 993.198460] pps_core [ 993.198460] wmi [ 993.198461] sunrpc [ 993.198461] dm_mirror [ 993.198462] dm_region_hash [ 993.198462] dm_log [ 993.198462] dm_mod [ 993.198463] iscsi_tcp [ 993.198463] libiscsi_tcp [ 993.198464] libiscsi [ 993.198464] scsi_transport_iscsi [ 993.198465] fuse [ 993.198465] [ 993.198468] CPU: 13 PID: 10825 Comm: mdt01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 993.198469] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 993.198470] task: ffff98322385d280 ti: ffff983225500000 task.ti: ffff983225500000 [ 993.198471] RIP: 0010:[] [ 993.198478] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 993.198479] RSP: 0018:ffff983225503a50 EFLAGS: 00000246 [ 993.198480] RAX: 0000000000000000 RBX: ffff9832bf340000 RCX: 0000000000690000 [ 993.198481] RDX: ffff97f2bfadb8c0 RSI: 0000000000190000 RDI: ffff97f2a83bcf40 [ 993.198482] RBP: ffff983225503a50 R08: ffff9832bf35b8c0 R09: 0000000000000000 [ 993.198483] R10: 0000000000000001 R11: 0000000000000005 R12: ffff983225503a18 [ 993.198484] R13: ffffffffa00ec9b1 R14: ffff9832255039f8 R15: ffff983225503a28 [ 993.198486] FS: 0000000000000000(0000) GS:ffff9832bf340000(0000) knlGS:0000000000000000 [ 993.198486] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 993.198487] CR2: 00002aaaab176a00 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 993.198489] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 993.198490] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 993.198490] Call Trace: [ 993.198497] [] queued_spin_lock_slowpath+0xb/0xf [ 993.198501] [] _raw_spin_lock+0x30/0x40 [ 993.198515] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 993.198536] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 993.198540] [] ? wake_up_atomic_t+0x30/0x30 [ 993.198548] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 993.198592] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 993.198614] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 993.198617] [] ? enqueue_task_fair+0x208/0x6c0 [ 993.198619] [] ? check_preempt_curr+0x80/0xa0 [ 993.198621] [] ? ttwu_do_wakeup+0x19/0x100 [ 993.198658] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 993.198687] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 993.198689] [] ? __getnstimeofday64+0x3f/0xd0 [ 993.198715] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 993.198741] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 993.198742] [] ? __wake_up_common_lock+0x91/0xc0 [ 993.198744] [] ? sched_feat_set+0xf0/0xf0 [ 993.198769] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 993.198793] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 993.198796] [] kthread+0xd1/0xe0 [ 993.198798] [] ? insert_kthread_work+0x40/0x40 [ 993.198801] [] ret_from_fork_nospec_begin+0x21/0x21 [ 993.198803] [] ? insert_kthread_work+0x40/0x40 [ 993.198803] Code: [ 993.198804] 0d [ 993.198804] 48 [ 993.198805] 98 [ 993.198805] 83 [ 993.198806] e2 [ 993.198806] 30 [ 993.198806] 48 [ 993.198807] 81 [ 993.198807] c2 [ 993.198807] c0 [ 993.198808] b8 [ 993.198808] 01 [ 993.198808] 00 [ 993.198809] 48 [ 993.198809] 03 [ 993.198809] 14 [ 993.198810] c5 [ 993.198810] e0 [ 993.198810] 17 [ 993.198811] d5 [ 993.198811] a0 [ 993.198811] 4c [ 993.198812] 89 [ 993.198812] 02 [ 993.198813] 41 [ 993.198813] 8b [ 993.198813] 40 [ 993.198814] 08 [ 993.198814] 85 [ 993.198814] c0 [ 993.198815] 75 [ 993.198815] 0f [ 993.198816] 0f [ 993.198816] 1f [ 993.198816] 44 [ 993.198817] 00 [ 993.198817] 00 [ 993.198817] f3 [ 993.198818] 90 [ 993.198818] 41 [ 993.198819] 8b [ 993.198819] 40 [ 993.198819] 08 [ 993.198820] <85> [ 993.198820] c0 [ 993.198820] 74 [ 993.198821] f6 [ 993.198821] 4d [ 993.198822] 8b [ 993.198822] 08 [ 993.198822] 4d [ 993.198823] 85 [ 993.198823] c9 [ 993.198823] 74 [ 993.198824] 04 [ 993.198824] 41 [ 993.198824] 0f [ 993.198825] 18 [ 993.198825] 09 [ 993.198826] 8b [ 993.198826] 17 [ 993.198826] 0f [ 993.198827] b7 [ 993.198827] c2 [ 993.198827] [ 994.616463] lod(OE) [ 994.619393] mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 [ 994.697562] cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 994.754187] CPU: 0 PID: 17320 Comm: mdt00_008 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 994.768453] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 994.780779] task: ffff98321989e300 ti: ffff98321cd1c000 task.ti: ffff98321cd1c000 [ 994.789615] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 994.801177] RSP: 0018:ffff98321cd1fab0 EFLAGS: 00000246 [ 994.807587] RAX: 0000000000000000 RBX: ffff97f2bfa00000 RCX: 0000000000010000 [ 994.816034] RDX: ffff9832bf55b8c0 RSI: 0000000000e90000 RDI: ffff97f2a83bcf40 [ 994.824482] RBP: ffff98321cd1fab0 R08: ffff97f2bfa1b8c0 R09: 0000000000000000 [ 994.832930] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321cd1fa78 [ 994.841377] R13: ffffffffa00ec9b1 R14: ffff98321cd1fa58 R15: ffff98321cd1fa88 [ 994.849826] FS: 0000000000000000(0000) GS:ffff97f2bfa00000(0000) knlGS:0000000000000000 [ 994.859340] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 994.866236] CR2: 00007ffff7ff8000 CR3: 0000007f0e586000 CR4: 00000000003607f0 [ 994.874685] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 994.883133] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 994.891580] Call Trace: [ 994.894792] [] queued_spin_lock_slowpath+0xb/0xf [ 994.902272] [] _raw_spin_lock+0x30/0x40 [ 994.908881] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 994.916553] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 994.925002] [] ? wake_up_atomic_t+0x30/0x30 [ 994.932001] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 994.939405] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 994.947683] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 994.955647] [] ? check_preempt_curr+0x80/0xa0 [ 994.962855] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 994.970745] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 994.978059] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 994.986244] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 994.995177] [] ? __getnstimeofday64+0x3f/0xd0 [ 995.002391] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 995.011456] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 995.019516] [] ? __wake_up_common_lock+0x91/0xc0 [ 995.026994] [] ? sched_feat_set+0xf0/0xf0 [ 995.033829] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 995.041311] [] ? __switch_to+0xce/0x5a0 [ 995.047955] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 995.056696] [] kthread+0xd1/0xe0 [ 995.062624] [] ? insert_kthread_work+0x40/0x40 [ 995.069909] [] ret_from_fork_nospec_begin+0x21/0x21 [ 995.077679] [] ? insert_kthread_work+0x40/0x40 [ 995.084962] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 995.608704] LustreError: 10850:0:(ldlm_lib.c:3348:target_bulk_io()) @@@ timeout on bulk READ after 100+1644341660s req@ffff983211e66050 x1723699781038784/t0(0) o256->fba94177-391f-c81c-2019-79d229bb5a3e@192.168.112.44@o2ib15:244/0 lens 304/240 e 2 to 0 dl 1644342679 ref 1 fl Interpret:/0/0 rc 0/0 [ 997.604939] LustreError: 18432:0:(ldlm_lib.c:3348:target_bulk_io()) @@@ timeout on bulk READ after 100+1644341662s req@ffff98321166b050 x1722512177538688/t0(0) o256->b9d7f2fe-0b82-672f-a97d-eb8233182993@192.168.129.31@o2ib36:246/0 lens 304/240 e 2 to 0 dl 1644342681 ref 1 fl Interpret:/0/0 rc 0/0 [ 1001.226313] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [mdt00_009:17321] [ 1001.235440] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1001.315245] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1001.376306] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1001.390668] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1001.402994] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 1001.411830] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1001.423390] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 1001.429801] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 1001.438250] RDX: ffff97f2bfadb8c0 RSI: 0000000000190000 RDI: ffff97f2a83bcf40 [ 1001.446697] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 1001.455145] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 1001.463592] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 1001.472039] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 1001.481552] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1001.488448] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1001.496894] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1001.505340] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1001.513786] Call Trace: [ 1001.517000] [] queued_spin_lock_slowpath+0xb/0xf [ 1001.524479] [] _raw_spin_lock+0x30/0x40 [ 1001.531090] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1001.538766] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1001.546736] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1001.555282] [] ? wake_up_atomic_t+0x30/0x30 [ 1001.562280] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1001.569693] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1001.577965] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1001.586528] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1001.594686] [] ? enqueue_task_fair+0x208/0x6c0 [ 1001.601971] [] ? sched_clock_cpu+0xa5/0xe0 [ 1001.608867] [] ? check_preempt_curr+0x80/0xa0 [ 1001.616055] [] ? ttwu_do_wakeup+0x19/0x100 [ 1001.622975] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1001.631153] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1001.640078] [] ? __getnstimeofday64+0x3f/0xd0 [ 1001.647286] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1001.656332] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1001.664391] [] ? __wake_up_common_lock+0x91/0xc0 [ 1001.671869] [] ? sched_feat_set+0xf0/0xf0 [ 1001.678687] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1001.686168] [] ? __switch_to+0xce/0x5a0 [ 1001.692790] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1001.701530] [] kthread+0xd1/0xe0 [ 1001.707458] [] ? insert_kthread_work+0x40/0x40 [ 1001.714743] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1001.722513] [] ? insert_kthread_work+0x40/0x40 [ 1001.729796] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1003.225588] LustreError: 8601:0:(ldlm_lib.c:3348:target_bulk_io()) @@@ timeout on bulk READ after 100+1644341660s req@ffff983211669850 x1716067274147904/t0(0) o256->eddbb2a2-aa13-5acf-e4aa-214a99cc275d@192.168.133.127@o2ib39:254/0 lens 304/240 e 2 to 0 dl 1644342689 ref 1 fl Interpret:/0/0 rc 0/0 [ 1003.805110] Lustre: MGS: haven't heard from client 1c26a8f6-69a7-3b18-7bfb-c94bc7a5bbab (at 192.168.128.35@o2ib31) in 227 seconds. I think it's dead, and I am evicting it. exp ffff98321c0da800, cur 1644342660 expire 1644342510 last 1644342433 [ 1003.829563] Lustre: Skipped 13 previous similar messages [ 1005.080756] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [mdt00_013:17594] [ 1005.089788] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1005.177768] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [mdt01_022:17601] [ 1005.169614] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx [ 1005.177769] Modules linked in: [ 1005.177771] osp(OE) [ 1005.177772] mdd(OE) [ 1005.177772] lod(OE) [ 1005.177773] mdt(OE) [ 1005.177773] lfsck(OE) [ 1005.177774] mgs(OE) [ 1005.177774] mgc(OE) [ 1005.177774] osd_zfs(OE) [ 1005.177775] lquota(OE) [ 1005.177775] fid(OE) [ 1005.177776] fld(OE) [ 1005.177776] ptlrpc(OE) [ 1005.177776] obdclass(OE) [ 1005.177777] ko2iblnd(OE) [ 1005.177777] lnet(OE) [ 1005.177777] libcfs(OE) [ 1005.177778] iTCO_wdt [ 1005.177779] iTCO_vendor_support [ 1005.177779] sb_edac [ 1005.177780] intel_powerclamp [ 1005.177780] coretemp [ 1005.177780] intel_rapl [ 1005.177781] iosf_mbi [ 1005.177781] rpcrdma [ 1005.177782] ib_iser [ 1005.177782] kvm [ 1005.177783] irqbypass [ 1005.177783] joydev [ 1005.177784] pcspkr [ 1005.177784] lpc_ich [ 1005.177785] i2c_i801 [ 1005.177785] ioatdma [ 1005.177785] ses [ 1005.177786] enclosure [ 1005.177786] sg [ 1005.177786] ipmi_si [ 1005.177787] ipmi_devintf [ 1005.177787] ipmi_msghandler [ 1005.177788] acpi_power_meter [ 1005.177788] dm_round_robin [ 1005.177789] acpi_cpufreq [ 1005.177789] ib_ipoib [ 1005.177790] rdma_ucm [ 1005.177790] ib_umad [ 1005.177791] sch_fq_codel [ 1005.177791] iw_cxgb4 [ 1005.177792] rdma_cm [ 1005.177792] iw_cm [ 1005.177793] ib_cm [ 1005.177793] iw_cxgb3 [ 1005.177794] zfs(POE) [ 1005.177795] binfmt_misc [ 1005.177795] zunicode(POE) [ 1005.177795] zavl(POE) [ 1005.177796] icp(POE) [ 1005.177796] zcommon(POE) [ 1005.177797] znvpair(POE) [ 1005.177797] spl(OE) [ 1005.177798] msr_safe(OE) [ 1005.177798] ip_tables [ 1005.177799] nfsv3 [ 1005.177799] nfs_acl [ 1005.177800] rpcsec_gss_krb5 [ 1005.177800] auth_rpcgss [ 1005.177801] nfsv4 [ 1005.177801] dns_resolver [ 1005.177802] nfs [ 1005.177802] lockd [ 1005.177802] grace [ 1005.177803] fscache [ 1005.177803] overlay(T) [ 1005.177804] ext4 [ 1005.177804] mbcache [ 1005.177805] jbd2 [ 1005.177806] dm_service_time [ 1005.177806] sd_mod [ 1005.177807] crc_t10dif [ 1005.177807] crct10dif_generic [ 1005.177807] mlx5_ib [ 1005.177808] ib_uverbs [ 1005.177808] ib_core [ 1005.177809] be2iscsi [ 1005.177809] bnx2i [ 1005.177810] cnic [ 1005.177810] uio [ 1005.177810] cxgb4i [ 1005.177811] cxgb4 [ 1005.177811] cxgb3i [ 1005.177812] cxgb3 [ 1005.177812] mdio [ 1005.177813] libcxgbi [ 1005.177813] libcxgb [ 1005.177813] qla4xxx [ 1005.177814] iscsi_boot_sysfs [ 1005.177814] mgag200 [ 1005.177815] i2c_algo_bit [ 1005.177815] 8021q [ 1005.177816] drm_kms_helper [ 1005.177816] garp [ 1005.177817] syscopyarea [ 1005.177817] mrp [ 1005.177818] crct10dif_pclmul [ 1005.177818] stp [ 1005.177819] sysfillrect [ 1005.177819] crct10dif_common [ 1005.177819] sysimgblt [ 1005.177820] crc32_pclmul [ 1005.177820] llc [ 1005.177821] fb_sys_fops [ 1005.177821] crc32c_intel [ 1005.177822] ttm [ 1005.177822] ghash_clmulni_intel [ 1005.177823] mlx5_core [ 1005.177823] drm [ 1005.177823] mxm_wmi [ 1005.177824] ahci [ 1005.177824] ixgbe(OE) [ 1005.177825] aesni_intel [ 1005.177825] mpt3sas [ 1005.177826] mlxfw [ 1005.177826] dm_multipath [ 1005.177826] lrw [ 1005.177827] devlink [ 1005.177827] libahci [ 1005.177827] gf128mul [ 1005.177828] dca [ 1005.177828] glue_helper [ 1005.177829] ablk_helper [ 1005.177829] raid_class [ 1005.177829] ptp [ 1005.177830] drm_panel_orientation_quirks [ 1005.177831] libata [ 1005.177831] cryptd [ 1005.177831] scsi_transport_sas [ 1005.177832] pps_core [ 1005.177832] wmi [ 1005.177833] sunrpc [ 1005.177833] dm_mirror [ 1005.177833] dm_region_hash [ 1005.177834] dm_log [ 1005.177834] dm_mod [ 1005.177835] iscsi_tcp [ 1005.177835] libiscsi_tcp [ 1005.177835] libiscsi [ 1005.177836] scsi_transport_iscsi [ 1005.177836] fuse [ 1005.177837] [ 1005.177839] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.177840] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.177840] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 1005.177841] RIP: 0010:[] [ 1005.177847] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1005.177848] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 1005.177849] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 1005.177850] RDX: ffff9832bf35b8c0 RSI: 0000000000690001 RDI: ffff97f2a83bcf40 [ 1005.177850] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 1005.177851] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 1005.177851] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 1005.177852] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 1005.177853] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.177854] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1005.177855] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.177855] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.177856] Call Trace: [ 1005.177862] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.177867] [] _raw_spin_lock+0x30/0x40 [ 1005.177877] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.177893] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.177896] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.177902] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.177940] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.177958] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1005.177978] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1005.177995] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1005.178012] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1005.178041] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1005.178068] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.178071] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.178096] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.178122] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.178124] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.178125] [] ? sched_feat_set+0xf0/0xf0 [ 1005.178151] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.178177] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.178179] [] kthread+0xd1/0xe0 [ 1005.178180] [] ? insert_kthread_work+0x40/0x40 [ 1005.178182] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.178184] [] ? insert_kthread_work+0x40/0x40 [ 1005.178184] Code: [ 1005.178185] 0d [ 1005.178185] 48 [ 1005.178185] 98 [ 1005.178185] 83 [ 1005.178186] e2 [ 1005.178186] 30 [ 1005.178186] 48 [ 1005.178186] 81 [ 1005.178187] c2 [ 1005.178187] c0 [ 1005.178187] b8 [ 1005.178187] 01 [ 1005.178188] 00 [ 1005.178188] 48 [ 1005.178188] 03 [ 1005.178189] 14 [ 1005.178189] c5 [ 1005.178189] e0 [ 1005.178190] 17 [ 1005.178190] d5 [ 1005.178190] a0 [ 1005.178190] 4c [ 1005.178191] 89 [ 1005.178191] 02 [ 1005.178191] 41 [ 1005.178191] 8b [ 1005.178192] 40 [ 1005.178192] 08 [ 1005.178192] 85 [ 1005.178193] c0 [ 1005.178193] 75 [ 1005.178193] 0f [ 1005.178193] 0f [ 1005.178194] 1f [ 1005.178194] 44 [ 1005.178194] 00 [ 1005.178195] 00 [ 1005.178195] f3 [ 1005.178195] 90 [ 1005.178195] 41 [ 1005.178196] 8b [ 1005.178196] 40 [ 1005.178196] 08 [ 1005.178197] <85> [ 1005.178197] c0 [ 1005.178197] 74 [ 1005.178197] f6 [ 1005.178198] 4d [ 1005.178198] 8b [ 1005.178198] 08 [ 1005.178198] 4d [ 1005.178199] 85 [ 1005.178199] c9 [ 1005.178199] 74 [ 1005.178199] 04 [ 1005.178200] 41 [ 1005.178200] 0f [ 1005.178200] 18 [ 1005.178201] 09 [ 1005.178201] 8b [ 1005.178201] 17 [ 1005.178201] 0f [ 1005.178202] b7 [ 1005.178202] c2 [ 1005.178202] [ 1005.204772] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [mdt01_016:17554] [ 1005.204773] Modules linked in: [ 1005.204773] osp(OE) [ 1005.204774] mdd(OE) [ 1005.204774] lod(OE) [ 1005.204775] mdt(OE) [ 1005.204775] lfsck(OE) [ 1005.204775] mgs(OE) [ 1005.204776] mgc(OE) [ 1005.204776] osd_zfs(OE) [ 1005.204777] lquota(OE) [ 1005.204777] fid(OE) [ 1005.204778] fld(OE) [ 1005.204778] ptlrpc(OE) [ 1005.204778] obdclass(OE) [ 1005.204779] ko2iblnd(OE) [ 1005.204779] lnet(OE) [ 1005.204780] libcfs(OE) [ 1005.204780] iTCO_wdt [ 1005.204781] iTCO_vendor_support [ 1005.204781] sb_edac [ 1005.204781] intel_powerclamp [ 1005.204782] coretemp [ 1005.204782] intel_rapl [ 1005.204782] iosf_mbi [ 1005.204783] rpcrdma [ 1005.204783] ib_iser [ 1005.204783] kvm [ 1005.204784] irqbypass [ 1005.204784] joydev [ 1005.204785] pcspkr [ 1005.204785] lpc_ich [ 1005.204785] i2c_i801 [ 1005.204786] ioatdma [ 1005.204786] ses [ 1005.204786] enclosure [ 1005.204787] sg [ 1005.204787] ipmi_si [ 1005.204787] ipmi_devintf [ 1005.204788] ipmi_msghandler [ 1005.204788] acpi_power_meter [ 1005.204789] dm_round_robin [ 1005.204789] acpi_cpufreq [ 1005.204789] ib_ipoib [ 1005.204790] rdma_ucm [ 1005.204790] ib_umad [ 1005.204790] sch_fq_codel [ 1005.204791] iw_cxgb4 [ 1005.204791] rdma_cm [ 1005.204792] iw_cm [ 1005.204792] ib_cm [ 1005.204792] iw_cxgb3 [ 1005.204793] zfs(POE) [ 1005.204793] binfmt_misc [ 1005.204794] zunicode(POE) [ 1005.204794] zavl(POE) [ 1005.204795] icp(POE) [ 1005.204795] zcommon(POE) [ 1005.204795] znvpair(POE) [ 1005.204796] spl(OE) [ 1005.204796] msr_safe(OE) [ 1005.204797] ip_tables [ 1005.204797] nfsv3 [ 1005.204797] nfs_acl [ 1005.204798] rpcsec_gss_krb5 [ 1005.204798] auth_rpcgss [ 1005.204799] nfsv4 [ 1005.204799] dns_resolver [ 1005.204800] nfs [ 1005.204800] lockd [ 1005.204800] grace [ 1005.204801] fscache [ 1005.204801] overlay(T) [ 1005.204802] ext4 [ 1005.204802] mbcache [ 1005.204803] jbd2 [ 1005.204803] dm_service_time [ 1005.204803] sd_mod [ 1005.204804] crc_t10dif [ 1005.204804] crct10dif_generic [ 1005.204805] mlx5_ib [ 1005.204805] ib_uverbs [ 1005.204805] ib_core [ 1005.204806] be2iscsi [ 1005.204806] bnx2i [ 1005.204806] cnic [ 1005.204807] uio [ 1005.204807] cxgb4i [ 1005.204808] cxgb4 [ 1005.204808] cxgb3i [ 1005.204808] cxgb3 [ 1005.204809] mdio [ 1005.204809] libcxgbi [ 1005.204810] libcxgb [ 1005.204810] qla4xxx [ 1005.204811] iscsi_boot_sysfs [ 1005.204811] mgag200 [ 1005.204811] i2c_algo_bit [ 1005.204812] 8021q [ 1005.204812] drm_kms_helper [ 1005.204813] garp [ 1005.204813] syscopyarea [ 1005.204813] mrp [ 1005.204814] crct10dif_pclmul [ 1005.204814] stp [ 1005.204814] sysfillrect [ 1005.204815] crct10dif_common [ 1005.204815] sysimgblt [ 1005.204816] crc32_pclmul [ 1005.204816] llc [ 1005.204817] fb_sys_fops [ 1005.204817] crc32c_intel [ 1005.204818] ttm [ 1005.204818] ghash_clmulni_intel [ 1005.204818] mlx5_core [ 1005.204819] drm [ 1005.204819] mxm_wmi [ 1005.204820] ahci [ 1005.204820] ixgbe(OE) [ 1005.204821] aesni_intel [ 1005.204821] mpt3sas [ 1005.204821] mlxfw [ 1005.204822] dm_multipath [ 1005.204822] lrw [ 1005.204822] devlink [ 1005.204823] libahci [ 1005.204823] gf128mul [ 1005.204824] dca [ 1005.204824] glue_helper [ 1005.204824] ablk_helper [ 1005.204825] raid_class [ 1005.204825] ptp [ 1005.204825] drm_panel_orientation_quirks [ 1005.204826] libata [ 1005.204826] cryptd [ 1005.204827] scsi_transport_sas [ 1005.204827] pps_core [ 1005.204828] wmi [ 1005.204828] sunrpc [ 1005.204828] dm_mirror [ 1005.204829] dm_region_hash [ 1005.204829] dm_log [ 1005.204829] dm_mod [ 1005.204830] iscsi_tcp [ 1005.204830] libiscsi_tcp [ 1005.204831] libiscsi [ 1005.204831] scsi_transport_iscsi [ 1005.204831] fuse [ 1005.204832] [ 1005.204834] CPU: 14 PID: 17554 Comm: mdt01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.204834] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.204835] task: ffff98321c985280 ti: ffff98321b544000 task.ti: ffff98321b544000 [ 1005.204836] RIP: 0010:[] [ 1005.204838] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1005.204839] RSP: 0018:ffff98321b547ab0 EFLAGS: 00000246 [ 1005.204840] RAX: 0000000000000000 RBX: ffff9832bf380000 RCX: 0000000000710000 [ 1005.204841] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 1005.204842] RBP: ffff98321b547ab0 R08: ffff9832bf39b8c0 R09: 0000000000000000 [ 1005.204843] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321b547a78 [ 1005.204844] R13: ffffffffa00ec9b1 R14: ffff98321b547a58 R15: ffff98321b547a88 [ 1005.204845] FS: 0000000000000000(0000) GS:ffff9832bf380000(0000) knlGS:0000000000000000 [ 1005.204847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.204848] CR2: 00007ffff7fd6000 CR3: 0000003f67e7a000 CR4: 00000000003607e0 [ 1005.204848] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.204849] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.204850] Call Trace: [ 1005.204852] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.204854] [] _raw_spin_lock+0x30/0x40 [ 1005.204860] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.204868] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.204870] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.204878] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.204902] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.204928] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1005.204930] [] ? check_preempt_curr+0x80/0xa0 [ 1005.204954] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1005.204978] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1005.205009] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1005.205037] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.205039] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.205066] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.205091] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.205093] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.205094] [] ? sched_feat_set+0xf0/0xf0 [ 1005.205118] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.205142] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.205144] [] kthread+0xd1/0xe0 [ 1005.205146] [] ? insert_kthread_work+0x40/0x40 [ 1005.205148] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.205149] [] ? insert_kthread_work+0x40/0x40 [ 1005.205150] Code: [ 1005.205151] 13 [ 1005.205151] 48 [ 1005.205151] c1 [ 1005.205152] ea [ 1005.205152] 0d [ 1005.205152] 48 [ 1005.205153] 98 [ 1005.205153] 83 [ 1005.205153] e2 [ 1005.205154] 30 [ 1005.205154] 48 [ 1005.205154] 81 [ 1005.205155] c2 [ 1005.205155] c0 [ 1005.205155] b8 [ 1005.205156] 01 [ 1005.205156] 00 [ 1005.205156] 48 [ 1005.205157] 03 [ 1005.205157] 14 [ 1005.205158] c5 [ 1005.205158] e0 [ 1005.205158] 17 [ 1005.205159] d5 [ 1005.205159] a0 [ 1005.205159] 4c [ 1005.205160] 89 [ 1005.205160] 02 [ 1005.205161] 41 [ 1005.205161] 8b [ 1005.205161] 40 [ 1005.205162] 08 [ 1005.205162] 85 [ 1005.205162] c0 [ 1005.205163] 75 [ 1005.205163] 0f [ 1005.205163] 0f [ 1005.205164] 1f [ 1005.205164] 44 [ 1005.205164] 00 [ 1005.205165] 00 [ 1005.205165] f3 [ 1005.205165] 90 [ 1005.205166] <41> [ 1005.205166] 8b [ 1005.205166] 40 [ 1005.205167] 08 [ 1005.205167] 85 [ 1005.205167] c0 [ 1005.205168] 74 [ 1005.205168] f6 [ 1005.205168] 4d [ 1005.205169] 8b [ 1005.205169] 08 [ 1005.205170] 4d [ 1005.205170] 85 [ 1005.205170] c9 [ 1005.205171] 74 [ 1005.205171] 04 [ 1005.205171] 41 [ 1005.205172] 0f [ 1005.205172] 18 [ 1005.205173] 09 [ 1005.205173] 8b [ 1005.205173] [ 1005.208771] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [mdt01_007:17310] [ 1005.208771] Modules linked in: [ 1005.208772] osp(OE) [ 1005.208773] mdd(OE) [ 1005.208773] lod(OE) [ 1005.208774] mdt(OE) [ 1005.208774] lfsck(OE) [ 1005.208774] mgs(OE) [ 1005.208775] mgc(OE) [ 1005.208775] osd_zfs(OE) [ 1005.208775] lquota(OE) [ 1005.208776] fid(OE) [ 1005.208776] fld(OE) [ 1005.208776] ptlrpc(OE) [ 1005.208777] obdclass(OE) [ 1005.208777] ko2iblnd(OE) [ 1005.208777] lnet(OE) [ 1005.208777] libcfs(OE) [ 1005.208778] iTCO_wdt [ 1005.208778] iTCO_vendor_support [ 1005.208778] sb_edac [ 1005.208779] intel_powerclamp [ 1005.208779] coretemp [ 1005.208779] intel_rapl [ 1005.208779] iosf_mbi [ 1005.208780] rpcrdma [ 1005.208780] ib_iser [ 1005.208781] kvm [ 1005.208781] irqbypass [ 1005.208781] joydev [ 1005.208781] pcspkr [ 1005.208782] lpc_ich [ 1005.208782] i2c_i801 [ 1005.208782] ioatdma [ 1005.208782] ses [ 1005.208783] enclosure [ 1005.208783] sg [ 1005.208783] ipmi_si [ 1005.208784] ipmi_devintf [ 1005.208784] ipmi_msghandler [ 1005.208784] acpi_power_meter [ 1005.208785] dm_round_robin [ 1005.208785] acpi_cpufreq [ 1005.208785] ib_ipoib [ 1005.208785] rdma_ucm [ 1005.208786] ib_umad [ 1005.208786] sch_fq_codel [ 1005.208786] iw_cxgb4 [ 1005.208787] rdma_cm [ 1005.208787] iw_cm [ 1005.208787] ib_cm [ 1005.208788] iw_cxgb3 [ 1005.208788] zfs(POE) [ 1005.208788] binfmt_misc [ 1005.208789] zunicode(POE) [ 1005.208789] zavl(POE) [ 1005.208789] icp(POE) [ 1005.208790] zcommon(POE) [ 1005.208790] znvpair(POE) [ 1005.208790] spl(OE) [ 1005.208790] msr_safe(OE) [ 1005.208791] ip_tables [ 1005.208791] nfsv3 [ 1005.208791] nfs_acl [ 1005.208792] rpcsec_gss_krb5 [ 1005.208792] auth_rpcgss [ 1005.208792] nfsv4 [ 1005.208792] dns_resolver [ 1005.208793] nfs [ 1005.208793] lockd [ 1005.208793] grace [ 1005.208794] fscache [ 1005.208794] overlay(T) [ 1005.208794] ext4 [ 1005.208794] mbcache [ 1005.208795] jbd2 [ 1005.208795] dm_service_time [ 1005.208795] sd_mod [ 1005.208796] crc_t10dif [ 1005.208796] crct10dif_generic [ 1005.208796] mlx5_ib [ 1005.208797] ib_uverbs [ 1005.208797] ib_core [ 1005.208797] be2iscsi [ 1005.208797] bnx2i [ 1005.208798] cnic [ 1005.208798] uio [ 1005.208798] cxgb4i [ 1005.208798] cxgb4 [ 1005.208799] cxgb3i [ 1005.208799] cxgb3 [ 1005.208799] mdio [ 1005.208799] libcxgbi [ 1005.208800] libcxgb [ 1005.208800] qla4xxx [ 1005.208800] iscsi_boot_sysfs [ 1005.208801] mgag200 [ 1005.208801] i2c_algo_bit [ 1005.208801] 8021q [ 1005.208801] drm_kms_helper [ 1005.208802] garp [ 1005.208802] syscopyarea [ 1005.208802] mrp [ 1005.208803] crct10dif_pclmul [ 1005.208803] stp [ 1005.208803] sysfillrect [ 1005.208803] crct10dif_common [ 1005.208804] sysimgblt [ 1005.208804] crc32_pclmul [ 1005.208804] llc [ 1005.208805] fb_sys_fops [ 1005.208805] crc32c_intel [ 1005.208805] ttm [ 1005.208806] ghash_clmulni_intel [ 1005.208806] mlx5_core [ 1005.208806] drm [ 1005.208806] mxm_wmi [ 1005.208807] ahci [ 1005.208807] ixgbe(OE) [ 1005.208807] aesni_intel [ 1005.208807] mpt3sas [ 1005.208808] mlxfw [ 1005.208808] dm_multipath [ 1005.208808] lrw [ 1005.208809] devlink [ 1005.208809] libahci [ 1005.208809] gf128mul [ 1005.208809] dca [ 1005.208810] glue_helper [ 1005.208810] ablk_helper [ 1005.208810] raid_class [ 1005.208810] ptp [ 1005.208811] drm_panel_orientation_quirks [ 1005.208811] libata [ 1005.208811] cryptd [ 1005.208812] scsi_transport_sas [ 1005.208812] pps_core [ 1005.208812] wmi [ 1005.208812] sunrpc [ 1005.208813] dm_mirror [ 1005.208813] dm_region_hash [ 1005.208813] dm_log [ 1005.208814] dm_mod [ 1005.208814] iscsi_tcp [ 1005.208814] libiscsi_tcp [ 1005.208815] libiscsi [ 1005.208815] scsi_transport_iscsi [ 1005.208815] fuse [ 1005.208816] [ 1005.208817] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.208818] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.208819] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 1005.208819] RIP: 0010:[] [ 1005.208821] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1005.208822] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 1005.208823] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 1005.208823] RDX: ffff97f2bfb5b8c0 RSI: 0000000000290001 RDI: ffff97f2a83bcf40 [ 1005.208824] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 1005.208825] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 1005.208825] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 1005.208826] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 1005.208827] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.208827] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 1005.208828] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.208829] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.208829] Call Trace: [ 1005.208831] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.208833] [] _raw_spin_lock+0x30/0x40 [ 1005.208838] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.208844] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.208846] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.208852] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.208870] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.208888] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1005.208890] [] ? check_preempt_curr+0x80/0xa0 [ 1005.208907] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1005.208924] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1005.208947] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1005.208969] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.208971] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.208990] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.209009] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.209010] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.209011] [] ? sched_feat_set+0xf0/0xf0 [ 1005.209029] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.209047] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.209049] [] kthread+0xd1/0xe0 [ 1005.209050] [] ? insert_kthread_work+0x40/0x40 [ 1005.209052] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.209053] [] ? insert_kthread_work+0x40/0x40 [ 1005.209053] Code: [ 1005.209054] 13 [ 1005.209054] 48 [ 1005.209054] c1 [ 1005.209055] ea [ 1005.209055] 0d [ 1005.209055] 48 [ 1005.209055] 98 [ 1005.209056] 83 [ 1005.209056] e2 [ 1005.209056] 30 [ 1005.209056] 48 [ 1005.209057] 81 [ 1005.209057] c2 [ 1005.209057] c0 [ 1005.209057] b8 [ 1005.209058] 01 [ 1005.209058] 00 [ 1005.209058] 48 [ 1005.209058] 03 [ 1005.209059] 14 [ 1005.209059] c5 [ 1005.209059] e0 [ 1005.209059] 17 [ 1005.209059] d5 [ 1005.209060] a0 [ 1005.209060] 4c [ 1005.209060] 89 [ 1005.209060] 02 [ 1005.209061] 41 [ 1005.209061] 8b [ 1005.209061] 40 [ 1005.209061] 08 [ 1005.209062] 85 [ 1005.209062] c0 [ 1005.209062] 75 [ 1005.209062] 0f [ 1005.209063] 0f [ 1005.209063] 1f [ 1005.209063] 44 [ 1005.209063] 00 [ 1005.209064] 00 [ 1005.209064] f3 [ 1005.209064] 90 [ 1005.209065] <41> [ 1005.209065] 8b [ 1005.209065] 40 [ 1005.209066] 08 [ 1005.209066] 85 [ 1005.209066] c0 [ 1005.209066] 74 [ 1005.209066] f6 [ 1005.209067] 4d [ 1005.209067] 8b [ 1005.209067] 08 [ 1005.209068] 4d [ 1005.209068] 85 [ 1005.209068] c9 [ 1005.209068] 74 [ 1005.209068] 04 [ 1005.209069] 41 [ 1005.209069] 0f [ 1005.209069] 18 [ 1005.209070] 09 [ 1005.209070] 8b [ 1005.209070] [ 1005.222773] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ll_mgs_0013:10822] [ 1005.222774] Modules linked in: [ 1005.222774] osp(OE) [ 1005.222775] mdd(OE) [ 1005.222775] lod(OE) [ 1005.222776] mdt(OE) [ 1005.222776] lfsck(OE) [ 1005.222776] mgs(OE) [ 1005.222777] mgc(OE) [ 1005.222777] osd_zfs(OE) [ 1005.222777] lquota(OE) [ 1005.222778] fid(OE) [ 1005.222778] fld(OE) [ 1005.222778] ptlrpc(OE) [ 1005.222779] obdclass(OE) [ 1005.222779] ko2iblnd(OE) [ 1005.222779] lnet(OE) [ 1005.222779] libcfs(OE) [ 1005.222780] iTCO_wdt [ 1005.222780] iTCO_vendor_support [ 1005.222780] sb_edac [ 1005.222781] intel_powerclamp [ 1005.222781] coretemp [ 1005.222781] intel_rapl [ 1005.222782] iosf_mbi [ 1005.222782] rpcrdma [ 1005.222782] ib_iser [ 1005.222783] kvm [ 1005.222783] irqbypass [ 1005.222783] joydev [ 1005.222783] pcspkr [ 1005.222784] lpc_ich [ 1005.222784] i2c_i801 [ 1005.222784] ioatdma [ 1005.222785] ses [ 1005.222785] enclosure [ 1005.222785] sg [ 1005.222785] ipmi_si [ 1005.222786] ipmi_devintf [ 1005.222786] ipmi_msghandler [ 1005.222786] acpi_power_meter [ 1005.222787] dm_round_robin [ 1005.222787] acpi_cpufreq [ 1005.222787] ib_ipoib [ 1005.222787] rdma_ucm [ 1005.222788] ib_umad [ 1005.222788] sch_fq_codel [ 1005.222788] iw_cxgb4 [ 1005.222788] rdma_cm [ 1005.222789] iw_cm [ 1005.222789] ib_cm [ 1005.222789] iw_cxgb3 [ 1005.222790] zfs(POE) [ 1005.222790] binfmt_misc [ 1005.222790] zunicode(POE) [ 1005.222791] zavl(POE) [ 1005.222791] icp(POE) [ 1005.222791] zcommon(POE) [ 1005.222792] znvpair(POE) [ 1005.222792] spl(OE) [ 1005.222792] msr_safe(OE) [ 1005.222792] ip_tables [ 1005.222793] nfsv3 [ 1005.222793] nfs_acl [ 1005.222793] rpcsec_gss_krb5 [ 1005.222794] auth_rpcgss [ 1005.222794] nfsv4 [ 1005.222794] dns_resolver [ 1005.222794] nfs [ 1005.222795] lockd [ 1005.222795] grace [ 1005.222795] fscache [ 1005.222796] overlay(T) [ 1005.222796] ext4 [ 1005.222796] mbcache [ 1005.222796] jbd2 [ 1005.222797] dm_service_time [ 1005.222797] sd_mod [ 1005.222797] crc_t10dif [ 1005.222797] crct10dif_generic [ 1005.222798] mlx5_ib [ 1005.222798] ib_uverbs [ 1005.222798] ib_core [ 1005.222799] be2iscsi [ 1005.222799] bnx2i [ 1005.222799] cnic [ 1005.222799] uio [ 1005.222800] cxgb4i [ 1005.222800] cxgb4 [ 1005.222800] cxgb3i [ 1005.222801] cxgb3 [ 1005.222801] mdio [ 1005.222801] libcxgbi [ 1005.222801] libcxgb [ 1005.222802] qla4xxx [ 1005.222802] iscsi_boot_sysfs [ 1005.222802] mgag200 [ 1005.222803] i2c_algo_bit [ 1005.222803] 8021q [ 1005.222803] drm_kms_helper [ 1005.222804] garp [ 1005.222804] syscopyarea [ 1005.222804] mrp [ 1005.222804] crct10dif_pclmul [ 1005.222805] stp [ 1005.222805] sysfillrect [ 1005.222805] crct10dif_common [ 1005.222806] sysimgblt [ 1005.222806] crc32_pclmul [ 1005.222806] llc [ 1005.222806] fb_sys_fops [ 1005.222807] crc32c_intel [ 1005.222807] ttm [ 1005.222807] ghash_clmulni_intel [ 1005.222808] mlx5_core [ 1005.222808] drm [ 1005.222808] mxm_wmi [ 1005.222808] ahci [ 1005.222809] ixgbe(OE) [ 1005.222809] aesni_intel [ 1005.222809] mpt3sas [ 1005.222809] mlxfw [ 1005.222810] dm_multipath [ 1005.222810] lrw [ 1005.222810] devlink [ 1005.222811] libahci [ 1005.222811] gf128mul [ 1005.222811] dca [ 1005.222811] glue_helper [ 1005.222812] ablk_helper [ 1005.222812] raid_class [ 1005.222812] ptp [ 1005.222813] drm_panel_orientation_quirks [ 1005.222813] libata [ 1005.222813] cryptd [ 1005.222813] scsi_transport_sas [ 1005.222814] pps_core [ 1005.222814] wmi [ 1005.222814] sunrpc [ 1005.222815] dm_mirror [ 1005.222815] dm_region_hash [ 1005.222815] dm_log [ 1005.222815] dm_mod [ 1005.222816] iscsi_tcp [ 1005.222816] libiscsi_tcp [ 1005.222816] libiscsi [ 1005.222817] scsi_transport_iscsi [ 1005.222817] fuse [ 1005.222817] [ 1005.222819] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.222819] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.222820] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 1005.222821] RIP: 0010:[] [ 1005.222823] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1005.222824] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 1005.222824] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 1005.222825] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 1005.222825] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 1005.222826] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 1005.222827] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 1005.222828] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 1005.222828] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.222829] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1005.222830] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.222830] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.222831] Call Trace: [ 1005.222833] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.222835] [] _raw_spin_lock+0x30/0x40 [ 1005.222841] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1005.222845] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1005.222852] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1005.222854] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.222859] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.222880] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.222896] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1005.222898] [] ? enqueue_task_fair+0x208/0x6c0 [ 1005.222899] [] ? check_preempt_curr+0x80/0xa0 [ 1005.222900] [] ? ttwu_do_wakeup+0x19/0x100 [ 1005.222924] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1005.222945] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.222947] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.222966] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.222985] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.222986] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.222987] [] ? sched_feat_set+0xf0/0xf0 [ 1005.223005] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.223007] [] ? __switch_to+0xce/0x5a0 [ 1005.223025] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.223026] [] kthread+0xd1/0xe0 [ 1005.223028] [] ? insert_kthread_work+0x40/0x40 [ 1005.223029] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.223030] [] ? insert_kthread_work+0x40/0x40 [ 1005.223031] Code: [ 1005.223031] 0d [ 1005.223032] 48 [ 1005.223032] 98 [ 1005.223032] 83 [ 1005.223032] e2 [ 1005.223033] 30 [ 1005.223033] 48 [ 1005.223033] 81 [ 1005.223033] c2 [ 1005.223034] c0 [ 1005.223034] b8 [ 1005.223034] 01 [ 1005.223034] 00 [ 1005.223035] 48 [ 1005.223035] 03 [ 1005.223035] 14 [ 1005.223035] c5 [ 1005.223036] e0 [ 1005.223036] 17 [ 1005.223036] d5 [ 1005.223036] a0 [ 1005.223037] 4c [ 1005.223037] 89 [ 1005.223037] 02 [ 1005.223037] 41 [ 1005.223038] 8b [ 1005.223038] 40 [ 1005.223038] 08 [ 1005.223038] 85 [ 1005.223038] c0 [ 1005.223039] 75 [ 1005.223039] 0f [ 1005.223039] 0f [ 1005.223039] 1f [ 1005.223040] 44 [ 1005.223040] 00 [ 1005.223040] 00 [ 1005.223041] f3 [ 1005.223041] 90 [ 1005.223041] 41 [ 1005.223041] 8b [ 1005.223042] 40 [ 1005.223042] 08 [ 1005.223042] <85> [ 1005.223042] c0 [ 1005.223043] 74 [ 1005.223043] f6 [ 1005.223043] 4d [ 1005.223043] 8b [ 1005.223044] 08 [ 1005.223044] 4d [ 1005.223044] 85 [ 1005.223044] c9 [ 1005.223045] 74 [ 1005.223045] 04 [ 1005.223045] 41 [ 1005.223045] 0f [ 1005.223046] 18 [ 1005.223046] 09 [ 1005.223046] 8b [ 1005.223046] 17 [ 1005.223047] 0f [ 1005.223047] b7 [ 1005.223047] c2 [ 1005.223047] [ 1005.228774] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ll_mgs_0014:10826] [ 1005.228774] Modules linked in: [ 1005.228775] osp(OE) [ 1005.228775] mdd(OE) [ 1005.228776] lod(OE) [ 1005.228776] mdt(OE) [ 1005.228776] lfsck(OE) [ 1005.228777] mgs(OE) [ 1005.228777] mgc(OE) [ 1005.228777] osd_zfs(OE) [ 1005.228778] lquota(OE) [ 1005.228778] fid(OE) [ 1005.228778] fld(OE) [ 1005.228778] ptlrpc(OE) [ 1005.228779] obdclass(OE) [ 1005.228779] ko2iblnd(OE) [ 1005.228779] lnet(OE) [ 1005.228780] libcfs(OE) [ 1005.228780] iTCO_wdt [ 1005.228780] iTCO_vendor_support [ 1005.228780] sb_edac [ 1005.228781] intel_powerclamp [ 1005.228781] coretemp [ 1005.228781] intel_rapl [ 1005.228782] iosf_mbi [ 1005.228782] rpcrdma [ 1005.228782] ib_iser [ 1005.228782] kvm [ 1005.228783] irqbypass [ 1005.228783] joydev [ 1005.228783] pcspkr [ 1005.228783] lpc_ich [ 1005.228784] i2c_i801 [ 1005.228784] ioatdma [ 1005.228784] ses [ 1005.228784] enclosure [ 1005.228785] sg [ 1005.228785] ipmi_si [ 1005.228785] ipmi_devintf [ 1005.228786] ipmi_msghandler [ 1005.228786] acpi_power_meter [ 1005.228786] dm_round_robin [ 1005.228786] acpi_cpufreq [ 1005.228787] ib_ipoib [ 1005.228787] rdma_ucm [ 1005.228787] ib_umad [ 1005.228788] sch_fq_codel [ 1005.228788] iw_cxgb4 [ 1005.228788] rdma_cm [ 1005.228788] iw_cm [ 1005.228789] ib_cm [ 1005.228789] iw_cxgb3 [ 1005.228789] zfs(POE) [ 1005.228789] binfmt_misc [ 1005.228790] zunicode(POE) [ 1005.228790] zavl(POE) [ 1005.228790] icp(POE) [ 1005.228791] zcommon(POE) [ 1005.228791] znvpair(POE) [ 1005.228791] spl(OE) [ 1005.228792] msr_safe(OE) [ 1005.228792] ip_tables [ 1005.228792] nfsv3 [ 1005.228793] nfs_acl [ 1005.228793] rpcsec_gss_krb5 [ 1005.228793] auth_rpcgss [ 1005.228793] nfsv4 [ 1005.228794] dns_resolver [ 1005.228794] nfs [ 1005.228794] lockd [ 1005.228794] grace [ 1005.228795] fscache [ 1005.228795] overlay(T) [ 1005.228795] ext4 [ 1005.228796] mbcache [ 1005.228796] jbd2 [ 1005.228796] dm_service_time [ 1005.228796] sd_mod [ 1005.228797] crc_t10dif [ 1005.228797] crct10dif_generic [ 1005.228797] mlx5_ib [ 1005.228797] ib_uverbs [ 1005.228798] ib_core [ 1005.228798] be2iscsi [ 1005.228798] bnx2i [ 1005.228799] cnic [ 1005.228799] uio [ 1005.228799] cxgb4i [ 1005.228799] cxgb4 [ 1005.228800] cxgb3i [ 1005.228800] cxgb3 [ 1005.228800] mdio [ 1005.228800] libcxgbi [ 1005.228801] libcxgb [ 1005.228801] qla4xxx [ 1005.228801] iscsi_boot_sysfs [ 1005.228801] mgag200 [ 1005.228802] i2c_algo_bit [ 1005.228802] 8021q [ 1005.228802] drm_kms_helper [ 1005.228803] garp [ 1005.228803] syscopyarea [ 1005.228803] mrp [ 1005.228803] crct10dif_pclmul [ 1005.228804] stp [ 1005.228804] sysfillrect [ 1005.228804] crct10dif_common [ 1005.228805] sysimgblt [ 1005.228805] crc32_pclmul [ 1005.228805] llc [ 1005.228806] fb_sys_fops [ 1005.228806] crc32c_intel [ 1005.228806] ttm [ 1005.228807] ghash_clmulni_intel [ 1005.228807] mlx5_core [ 1005.228807] drm [ 1005.228807] mxm_wmi [ 1005.228808] ahci [ 1005.228808] ixgbe(OE) [ 1005.228808] aesni_intel [ 1005.228809] mpt3sas [ 1005.228809] mlxfw [ 1005.228809] dm_multipath [ 1005.228809] lrw [ 1005.228810] devlink [ 1005.228810] libahci [ 1005.228810] gf128mul [ 1005.228811] dca [ 1005.228811] glue_helper [ 1005.228811] ablk_helper [ 1005.228811] raid_class [ 1005.228812] ptp [ 1005.228812] drm_panel_orientation_quirks [ 1005.228812] libata [ 1005.228812] cryptd [ 1005.228813] scsi_transport_sas [ 1005.228813] pps_core [ 1005.228813] wmi [ 1005.228814] sunrpc [ 1005.228814] dm_mirror [ 1005.228814] dm_region_hash [ 1005.228814] dm_log [ 1005.228815] dm_mod [ 1005.228815] iscsi_tcp [ 1005.228815] libiscsi_tcp [ 1005.228816] libiscsi [ 1005.228816] scsi_transport_iscsi [ 1005.228816] fuse [ 1005.228816] [ 1005.228818] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.228818] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.228819] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 1005.228820] RIP: 0010:[] [ 1005.228822] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1005.228822] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 1005.228823] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 1005.228824] RDX: ffff97f2bfa1b8c0 RSI: 0000000000010000 RDI: ffff97f2a83bcf40 [ 1005.228824] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 1005.228825] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 1005.228825] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 1005.228826] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 1005.228827] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.228828] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1005.228828] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.228829] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.228829] Call Trace: [ 1005.228832] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.228833] [] _raw_spin_lock+0x30/0x40 [ 1005.228839] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.228845] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.228847] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.228853] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.228871] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.228889] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1005.228890] [] ? check_preempt_curr+0x80/0xa0 [ 1005.228908] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1005.228925] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1005.228948] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1005.228969] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.228970] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.228990] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.229008] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.229010] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.229011] [] ? sched_feat_set+0xf0/0xf0 [ 1005.229029] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.229031] [] ? __switch_to+0xce/0x5a0 [ 1005.229049] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.229050] [] kthread+0xd1/0xe0 [ 1005.229052] [] ? insert_kthread_work+0x40/0x40 [ 1005.229053] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.229054] [] ? insert_kthread_work+0x40/0x40 [ 1005.229055] Code: [ 1005.229055] 0d [ 1005.229055] 48 [ 1005.229056] 98 [ 1005.229056] 83 [ 1005.229056] e2 [ 1005.229056] 30 [ 1005.229057] 48 [ 1005.229057] 81 [ 1005.229057] c2 [ 1005.229057] c0 [ 1005.229058] b8 [ 1005.229058] 01 [ 1005.229058] 00 [ 1005.229058] 48 [ 1005.229059] 03 [ 1005.229059] 14 [ 1005.229059] c5 [ 1005.229060] e0 [ 1005.229060] 17 [ 1005.229060] d5 [ 1005.229060] a0 [ 1005.229061] 4c [ 1005.229061] 89 [ 1005.229061] 02 [ 1005.229061] 41 [ 1005.229062] 8b [ 1005.229062] 40 [ 1005.229062] 08 [ 1005.229062] 85 [ 1005.229063] c0 [ 1005.229063] 75 [ 1005.229063] 0f [ 1005.229063] 0f [ 1005.229064] 1f [ 1005.229064] 44 [ 1005.229064] 00 [ 1005.229064] 00 [ 1005.229065] f3 [ 1005.229065] 90 [ 1005.229065] 41 [ 1005.229065] 8b [ 1005.229066] 40 [ 1005.229066] 08 [ 1005.229066] <85> [ 1005.229067] c0 [ 1005.229067] 74 [ 1005.229067] f6 [ 1005.229067] 4d [ 1005.229068] 8b [ 1005.229068] 08 [ 1005.229068] 4d [ 1005.229068] 85 [ 1005.229069] c9 [ 1005.229069] 74 [ 1005.229069] 04 [ 1005.229069] 41 [ 1005.229070] 0f [ 1005.229070] 18 [ 1005.229070] 09 [ 1005.229070] 8b [ 1005.229071] 17 [ 1005.229071] 0f [ 1005.229071] b7 [ 1005.229071] c2 [ 1005.229072] [ 1005.240775] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 23s! [mdt01_014:17318] [ 1005.240775] Modules linked in: [ 1005.240776] osp(OE) [ 1005.240777] mdd(OE) [ 1005.240777] lod(OE) [ 1005.240777] mdt(OE) [ 1005.240778] lfsck(OE) [ 1005.240778] mgs(OE) [ 1005.240778] mgc(OE) [ 1005.240778] osd_zfs(OE) [ 1005.240779] lquota(OE) [ 1005.240779] fid(OE) [ 1005.240779] fld(OE) [ 1005.240780] ptlrpc(OE) [ 1005.240780] obdclass(OE) [ 1005.240780] ko2iblnd(OE) [ 1005.240781] lnet(OE) [ 1005.240781] libcfs(OE) [ 1005.240781] iTCO_wdt [ 1005.240781] iTCO_vendor_support [ 1005.240782] sb_edac [ 1005.240782] intel_powerclamp [ 1005.240782] coretemp [ 1005.240783] intel_rapl [ 1005.240783] iosf_mbi [ 1005.240783] rpcrdma [ 1005.240783] ib_iser [ 1005.240784] kvm [ 1005.240784] irqbypass [ 1005.240784] joydev [ 1005.240784] pcspkr [ 1005.240785] lpc_ich [ 1005.240785] i2c_i801 [ 1005.240785] ioatdma [ 1005.240786] ses [ 1005.240786] enclosure [ 1005.240786] sg [ 1005.240786] ipmi_si [ 1005.240787] ipmi_devintf [ 1005.240787] ipmi_msghandler [ 1005.240787] acpi_power_meter [ 1005.240788] dm_round_robin [ 1005.240788] acpi_cpufreq [ 1005.240788] ib_ipoib [ 1005.240788] rdma_ucm [ 1005.240789] ib_umad [ 1005.240789] sch_fq_codel [ 1005.240789] iw_cxgb4 [ 1005.240790] rdma_cm [ 1005.240790] iw_cm [ 1005.240790] ib_cm [ 1005.240790] iw_cxgb3 [ 1005.240791] zfs(POE) [ 1005.240791] binfmt_misc [ 1005.240791] zunicode(POE) [ 1005.240792] zavl(POE) [ 1005.240792] icp(POE) [ 1005.240792] zcommon(POE) [ 1005.240793] znvpair(POE) [ 1005.240793] spl(OE) [ 1005.240793] msr_safe(OE) [ 1005.240794] ip_tables [ 1005.240794] nfsv3 [ 1005.240794] nfs_acl [ 1005.240795] rpcsec_gss_krb5 [ 1005.240795] auth_rpcgss [ 1005.240795] nfsv4 [ 1005.240796] dns_resolver [ 1005.240796] nfs [ 1005.240796] lockd [ 1005.240797] grace [ 1005.240797] fscache [ 1005.240797] overlay(T) [ 1005.240798] ext4 [ 1005.240798] mbcache [ 1005.240798] jbd2 [ 1005.240798] dm_service_time [ 1005.240799] sd_mod [ 1005.240799] crc_t10dif [ 1005.240800] crct10dif_generic [ 1005.240800] mlx5_ib [ 1005.240800] ib_uverbs [ 1005.240800] ib_core [ 1005.240801] be2iscsi [ 1005.240801] bnx2i [ 1005.240801] cnic [ 1005.240801] uio [ 1005.240802] cxgb4i [ 1005.240802] cxgb4 [ 1005.240802] cxgb3i [ 1005.240803] cxgb3 [ 1005.240803] mdio [ 1005.240803] libcxgbi [ 1005.240803] libcxgb [ 1005.240804] qla4xxx [ 1005.240804] iscsi_boot_sysfs [ 1005.240804] mgag200 [ 1005.240805] i2c_algo_bit [ 1005.240805] 8021q [ 1005.240805] drm_kms_helper [ 1005.240805] garp [ 1005.240806] syscopyarea [ 1005.240806] mrp [ 1005.240806] crct10dif_pclmul [ 1005.240807] stp [ 1005.240807] sysfillrect [ 1005.240807] crct10dif_common [ 1005.240807] sysimgblt [ 1005.240808] crc32_pclmul [ 1005.240808] llc [ 1005.240808] fb_sys_fops [ 1005.240808] crc32c_intel [ 1005.240809] ttm [ 1005.240809] ghash_clmulni_intel [ 1005.240809] mlx5_core [ 1005.240810] drm [ 1005.240810] mxm_wmi [ 1005.240810] ahci [ 1005.240810] ixgbe(OE) [ 1005.240811] aesni_intel [ 1005.240811] mpt3sas [ 1005.240811] mlxfw [ 1005.240812] dm_multipath [ 1005.240812] lrw [ 1005.240812] devlink [ 1005.240812] libahci [ 1005.240813] gf128mul [ 1005.240813] dca [ 1005.240813] glue_helper [ 1005.240814] ablk_helper [ 1005.240814] raid_class [ 1005.240814] ptp [ 1005.240815] drm_panel_orientation_quirks [ 1005.240815] libata [ 1005.240815] cryptd [ 1005.240816] scsi_transport_sas [ 1005.240816] pps_core [ 1005.240816] wmi [ 1005.240816] sunrpc [ 1005.240817] dm_mirror [ 1005.240817] dm_region_hash [ 1005.240817] dm_log [ 1005.240818] dm_mod [ 1005.240818] iscsi_tcp [ 1005.240818] libiscsi_tcp [ 1005.240819] libiscsi [ 1005.240819] scsi_transport_iscsi [ 1005.240819] fuse [ 1005.240819] [ 1005.240821] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.240822] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.240823] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 1005.240823] RIP: 0010:[] [ 1005.240825] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1005.240826] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 1005.240827] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 1005.240827] RDX: ffff97f2bfadb8c0 RSI: 0000000000190000 RDI: ffff97f2a83bcf40 [ 1005.240828] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 1005.240829] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 1005.240829] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 1005.240830] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 1005.240831] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.240831] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1005.240832] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.240833] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.240833] Call Trace: [ 1005.240835] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.240837] [] _raw_spin_lock+0x30/0x40 [ 1005.240842] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.240848] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.240850] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.240856] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.240874] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.240893] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1005.240912] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1005.240929] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1005.240946] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1005.240968] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1005.240990] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.240991] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.241010] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.241029] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.241030] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.241031] [] ? sched_feat_set+0xf0/0xf0 [ 1005.241049] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.241068] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.241070] [] kthread+0xd1/0xe0 [ 1005.241071] [] ? insert_kthread_work+0x40/0x40 [ 1005.241072] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.241074] [] ? insert_kthread_work+0x40/0x40 [ 1005.241074] Code: [ 1005.241075] 0d [ 1005.241075] 48 [ 1005.241075] 98 [ 1005.241075] 83 [ 1005.241076] e2 [ 1005.241076] 30 [ 1005.241076] 48 [ 1005.241076] 81 [ 1005.241077] c2 [ 1005.241077] c0 [ 1005.241077] b8 [ 1005.241077] 01 [ 1005.241078] 00 [ 1005.241078] 48 [ 1005.241078] 03 [ 1005.241079] 14 [ 1005.241079] c5 [ 1005.241079] e0 [ 1005.241079] 17 [ 1005.241080] d5 [ 1005.241080] a0 [ 1005.241080] 4c [ 1005.241080] 89 [ 1005.241081] 02 [ 1005.241081] 41 [ 1005.241081] 8b [ 1005.241082] 40 [ 1005.241082] 08 [ 1005.241082] 85 [ 1005.241082] c0 [ 1005.241083] 75 [ 1005.241083] 0f [ 1005.241083] 0f [ 1005.241083] 1f [ 1005.241084] 44 [ 1005.241084] 00 [ 1005.241084] 00 [ 1005.241085] f3 [ 1005.241085] 90 [ 1005.241085] 41 [ 1005.241086] 8b [ 1005.241086] 40 [ 1005.241086] 08 [ 1005.241086] <85> [ 1005.241087] c0 [ 1005.241087] 74 [ 1005.241087] f6 [ 1005.241088] 4d [ 1005.241088] 8b [ 1005.241088] 08 [ 1005.241088] 4d [ 1005.241089] 85 [ 1005.241089] c9 [ 1005.241089] 74 [ 1005.241089] 04 [ 1005.241089] 41 [ 1005.241090] 0f [ 1005.241090] 18 [ 1005.241090] 09 [ 1005.241090] 8b [ 1005.241091] 17 [ 1005.241091] 0f [ 1005.241091] b7 [ 1005.241091] c2 [ 1005.241092] [ 1005.245775] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [mdt01_017:17555] [ 1005.245776] Modules linked in: [ 1005.245776] osp(OE) [ 1005.245777] mdd(OE) [ 1005.245777] lod(OE) [ 1005.245777] mdt(OE) [ 1005.245778] lfsck(OE) [ 1005.245778] mgs(OE) [ 1005.245779] mgc(OE) [ 1005.245779] osd_zfs(OE) [ 1005.245780] lquota(OE) [ 1005.245780] fid(OE) [ 1005.245780] fld(OE) [ 1005.245781] ptlrpc(OE) [ 1005.245781] obdclass(OE) [ 1005.245782] ko2iblnd(OE) [ 1005.245782] lnet(OE) [ 1005.245783] libcfs(OE) [ 1005.245783] iTCO_wdt [ 1005.245784] iTCO_vendor_support [ 1005.245784] sb_edac [ 1005.245784] intel_powerclamp [ 1005.245785] coretemp [ 1005.245785] intel_rapl [ 1005.245785] iosf_mbi [ 1005.245786] rpcrdma [ 1005.245786] ib_iser [ 1005.245787] kvm [ 1005.245787] irqbypass [ 1005.245787] joydev [ 1005.245788] pcspkr [ 1005.245788] lpc_ich [ 1005.245788] i2c_i801 [ 1005.245789] ioatdma [ 1005.245789] ses [ 1005.245789] enclosure [ 1005.245790] sg [ 1005.245790] ipmi_si [ 1005.245791] ipmi_devintf [ 1005.245791] ipmi_msghandler [ 1005.245791] acpi_power_meter [ 1005.245792] dm_round_robin [ 1005.245792] acpi_cpufreq [ 1005.245793] ib_ipoib [ 1005.245793] rdma_ucm [ 1005.245793] ib_umad [ 1005.245794] sch_fq_codel [ 1005.245794] iw_cxgb4 [ 1005.245794] rdma_cm [ 1005.245795] iw_cm [ 1005.245795] ib_cm [ 1005.245795] iw_cxgb3 [ 1005.245796] zfs(POE) [ 1005.245796] binfmt_misc [ 1005.245797] zunicode(POE) [ 1005.245797] zavl(POE) [ 1005.245798] icp(POE) [ 1005.245798] zcommon(POE) [ 1005.245799] znvpair(POE) [ 1005.245799] spl(OE) [ 1005.245800] msr_safe(OE) [ 1005.245800] ip_tables [ 1005.245800] nfsv3 [ 1005.245801] nfs_acl [ 1005.245801] rpcsec_gss_krb5 [ 1005.245802] auth_rpcgss [ 1005.245802] nfsv4 [ 1005.245803] dns_resolver [ 1005.245803] nfs [ 1005.245803] lockd [ 1005.245804] grace [ 1005.245804] fscache [ 1005.245805] overlay(T) [ 1005.245805] ext4 [ 1005.245806] mbcache [ 1005.245806] jbd2 [ 1005.245806] dm_service_time [ 1005.245807] sd_mod [ 1005.245807] crc_t10dif [ 1005.245808] crct10dif_generic [ 1005.245808] mlx5_ib [ 1005.245808] ib_uverbs [ 1005.245809] ib_core [ 1005.245809] be2iscsi [ 1005.245810] bnx2i [ 1005.245810] cnic [ 1005.245810] uio [ 1005.245811] cxgb4i [ 1005.245811] cxgb4 [ 1005.245812] cxgb3i [ 1005.245812] cxgb3 [ 1005.245812] mdio [ 1005.245813] libcxgbi [ 1005.245813] libcxgb [ 1005.245814] qla4xxx [ 1005.245814] iscsi_boot_sysfs [ 1005.245814] mgag200 [ 1005.245815] i2c_algo_bit [ 1005.245815] 8021q [ 1005.245816] drm_kms_helper [ 1005.245816] garp [ 1005.245816] syscopyarea [ 1005.245817] mrp [ 1005.245817] crct10dif_pclmul [ 1005.245817] stp [ 1005.245818] sysfillrect [ 1005.245818] crct10dif_common [ 1005.245819] sysimgblt [ 1005.245819] crc32_pclmul [ 1005.245820] llc [ 1005.245820] fb_sys_fops [ 1005.245820] crc32c_intel [ 1005.245821] ttm [ 1005.245821] ghash_clmulni_intel [ 1005.245822] mlx5_core [ 1005.245822] drm [ 1005.245822] mxm_wmi [ 1005.245823] ahci [ 1005.245823] ixgbe(OE) [ 1005.245824] aesni_intel [ 1005.245824] mpt3sas [ 1005.245825] mlxfw [ 1005.245825] dm_multipath [ 1005.245825] lrw [ 1005.245826] devlink [ 1005.245826] libahci [ 1005.245827] gf128mul [ 1005.245827] dca [ 1005.245827] glue_helper [ 1005.245828] ablk_helper [ 1005.245828] raid_class [ 1005.245828] ptp [ 1005.245829] drm_panel_orientation_quirks [ 1005.245829] libata [ 1005.245830] cryptd [ 1005.245830] scsi_transport_sas [ 1005.245831] pps_core [ 1005.245831] wmi [ 1005.245831] sunrpc [ 1005.245832] dm_mirror [ 1005.245832] dm_region_hash [ 1005.245833] dm_log [ 1005.245833] dm_mod [ 1005.245833] iscsi_tcp [ 1005.245834] libiscsi_tcp [ 1005.245834] libiscsi [ 1005.245835] scsi_transport_iscsi [ 1005.245835] fuse [ 1005.245835] [ 1005.245837] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.245838] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.245839] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 1005.245840] RIP: 0010:[] [ 1005.245842] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1005.245843] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000246 [ 1005.245844] RAX: 0000000000000000 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 1005.245844] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 1005.245845] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: 0000000000000000 [ 1005.245846] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 1005.245847] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 1005.245848] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 1005.245850] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.245851] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1005.245852] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.245853] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.245853] Call Trace: [ 1005.245855] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.245857] [] _raw_spin_lock+0x30/0x40 [ 1005.245863] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.245871] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.245873] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.245881] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.245905] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.245929] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1005.245931] [] ? check_preempt_curr+0x80/0xa0 [ 1005.245954] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1005.245977] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1005.246007] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1005.246035] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.246037] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.246063] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.246087] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.246090] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.246151] iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1005.246153] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.246153] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.246154] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 1005.246157] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 1005.246157] RSP: 0018:ffff98321b877a50 EFLAGS: 00000246 [ 1005.246158] RAX: 0000000000000000 RBX: ffff97f2bfa80000 RCX: 0000000000110000 [ 1005.246159] RDX: ffff9832bf4db8c0 RSI: 0000000000d90000 RDI: ffff97f2a83bcf40 [ 1005.246159] RBP: ffff98321b877a50 R08: ffff97f2bfa9b8c0 R09: 0000000000000000 [ 1005.246160] R10: 0000000000000001 R11: 000000000000242e R12: ffff98321b877a18 [ 1005.246160] R13: ffff97f2bfc9ad40 R14: ffffffffa00f0602 R15: ffff98321b877a28 [ 1005.246161] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 1005.246162] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.246163] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1005.246163] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.246164] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.246164] Call Trace: [ 1005.246166] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.246168] [] _raw_spin_lock+0x30/0x40 [ 1005.246173] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.246179] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.246181] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.246187] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.246205] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.246222] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1005.246240] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1005.246241] [] ? enqueue_task_fair+0x208/0x6c0 [ 1005.246243] [] ? sched_clock_cpu+0xa5/0xe0 [ 1005.246244] [] ? check_preempt_curr+0x80/0xa0 [ 1005.246245] [] ? ttwu_do_wakeup+0x19/0x100 [ 1005.246268] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1005.246290] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.246291] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.246311] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.246331] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.246332] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.246333] [] ? sched_feat_set+0xf0/0xf0 [ 1005.246352] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.246353] [] ? __switch_to+0xce/0x5a0 [ 1005.246371] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.246373] [] kthread+0xd1/0xe0 [ 1005.246374] [] ? insert_kthread_work+0x40/0x40 [ 1005.246375] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.246377] [] ? insert_kthread_work+0x40/0x40 [ 1005.246390] Code: c1 e8 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 [ 1005.247775] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [mdt01_018:17592] [ 1005.247804] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1005.247824] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1005.247826] CPU: 30 PID: 17592 Comm: mdt01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1005.247827] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1005.247828] task: ffff983230579080 ti: ffff9832469b0000 task.ti: ffff9832469b0000 [ 1005.247831] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1005.247832] RSP: 0018:ffff9832469b3a50 EFLAGS: 00000246 [ 1005.247832] RAX: 0000000000000000 RBX: ffff9832bf580000 RCX: 0000000000f10000 [ 1005.247833] RDX: ffff9832bf21b8c0 RSI: 0000000000410000 RDI: ffff97f2a83bcf40 [ 1005.247834] RBP: ffff9832469b3a50 R08: ffff9832bf59b8c0 R09: 0000000000000000 [ 1005.247835] R10: 0000000000000002 R11: ffff983211336600 R12: ffff9832469b3a18 [ 1005.247836] R13: ffffffffa00ec9b1 R14: ffff9832469b39f8 R15: ffff9832469b3a28 [ 1005.247837] FS: 0000000000000000(0000) GS:ffff9832bf580000(0000) knlGS:0000000000000000 [ 1005.247838] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.247839] CR2: 00007ffff7fc5010 CR3: 0000003f74514000 CR4: 00000000003607e0 [ 1005.247839] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.247840] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.247841] Call Trace: [ 1005.247843] [] queued_spin_lock_slowpath+0xb/0xf [ 1005.247845] [] _raw_spin_lock+0x30/0x40 [ 1005.247851] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1005.247859] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1005.247861] [] ? wake_up_atomic_t+0x30/0x30 [ 1005.247869] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1005.247893] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1005.247916] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1005.247918] [] ? check_preempt_curr+0x80/0xa0 [ 1005.247920] [] ? ttwu_do_wakeup+0x19/0x100 [ 1005.247950] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1005.247978] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1005.247980] [] ? __getnstimeofday64+0x3f/0xd0 [ 1005.248006] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1005.248030] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1005.248031] [] ? __wake_up_common_lock+0x91/0xc0 [ 1005.248032] [] ? sched_feat_set+0xf0/0xf0 [ 1005.248057] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1005.248081] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1005.248083] [] kthread+0xd1/0xe0 [ 1005.248085] [] ? insert_kthread_work+0x40/0x40 [ 1005.248087] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1005.248088] [] ? insert_kthread_work+0x40/0x40 [ 1005.248107] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 1005.561828] LustreError: 13746:0:(ldlm_lib.c:3348:target_bulk_io()) @@@ timeout on bulk READ after 100+1644341658s req@ffff983211e65850 x1716102670219136/t0(0) o256->a4ab4643-04a8-2240-45a6-ef606746783c@192.168.130.54@o2ib39:232/0 lens 304/240 e 1 to 0 dl 1644342667 ref 1 fl Interpret:/0/0 rc 0/0 [ 1009.543212] [] ? sched_feat_set+0xf0/0xf0 [ 1009.550039] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1009.559064] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1009.569334] [] kthread+0xd1/0xe0 [ 1009.575263] [] ? insert_kthread_work+0x40/0x40 [ 1009.584094] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1009.591865] [] ? insert_kthread_work+0x40/0x40 [ 1009.600696] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 1009.623453] LNet: Service thread pid 16164 was inactive for 201.02s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [ 1009.623613] LNet: Service thread pid 15938 completed after 201.05s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [ 1009.623615] LNet: Skipped 5 previous similar messages [ 1009.624113] Lustre: 16173:0:(service.c:2169:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (69:132s); client may timeout. req@ffff983213487080 x1716155570630016/t468151437544(0) o101->b982d4e3-7968-8026-85af-117246f96b60@192.168.138.14@o2ib27:98/0 lens 1440/624 e 0 to 0 dl 1644342533 ref 1 fl Complete:/0/0 rc 0/0 [ 1009.699463] LNet: Skipped 1 previous similar message [ 1009.707034] LustreError: dumping log to /tmp/lustre-log.1644342665.16164 [ 1014.489878] LustreError: dumping log to /tmp/lustre-log.1644342670.10831 [ 1017.214154] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [mdt00_031:18467] [ 1017.218155] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [mdt00_030:16183] [ 1017.218187] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1017.218210] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1017.218212] CPU: 17 PID: 16183 Comm: mdt00_030 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1017.218212] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1017.218213] task: ffff97f250d53180 ti: ffff97f250d78000 task.ti: ffff97f250d78000 [ 1017.218218] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1017.218219] RSP: 0018:ffff97f250d7bab0 EFLAGS: 00000246 [ 1017.218219] RAX: 0000000000000000 RBX: ffff97f2bfc40000 RCX: 0000000000890000 [ 1017.218220] RDX: ffff9832bf39b8c0 RSI: 0000000000710000 RDI: ffff97f2a83bcf40 [ 1017.218221] RBP: ffff97f250d7bab0 R08: ffff97f2bfc5b8c0 R09: 0000000000000000 [ 1017.218221] R10: 0000000000000001 R11: 0000000000000400 R12: ffff97f250d7ba78 [ 1017.218222] R13: ffffffffa00ec9b1 R14: ffff97f250d7ba58 R15: ffff97f250d7ba88 [ 1017.218223] FS: 0000000000000000(0000) GS:ffff97f2bfc40000(0000) knlGS:0000000000000000 [ 1017.218224] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1017.218224] CR2: 00007ffff75f34b1 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1017.218225] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1017.218226] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1017.218226] Call Trace: [ 1017.218230] [] queued_spin_lock_slowpath+0xb/0xf [ 1017.218232] [] _raw_spin_lock+0x30/0x40 [ 1017.218243] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1017.218255] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1017.218257] [] ? wake_up_atomic_t+0x30/0x30 [ 1017.218263] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1017.218305] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1017.218324] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1017.218326] [] ? check_preempt_curr+0x80/0xa0 [ 1017.218343] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1017.218360] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1017.218389] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1017.218410] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1017.218412] [] ? __getnstimeofday64+0x3f/0xd0 [ 1017.218432] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1017.218450] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1017.218451] [] ? __wake_up_common_lock+0x91/0xc0 [ 1017.218452] [] ? sched_feat_set+0xf0/0xf0 [ 1017.218471] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1017.218473] [] ? __switch_to+0xce/0x5a0 [ 1017.218491] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1017.218493] [] kthread+0xd1/0xe0 [ 1017.218494] [] ? insert_kthread_work+0x40/0x40 [ 1017.218497] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1017.218498] [] ? insert_kthread_work+0x40/0x40 [ 1017.218512] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1017.723913] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1017.803767] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1017.864848] CPU: 16 PID: 18467 Comm: mdt00_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1017.879211] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1017.891539] task: ffff97f249f7a100 ti: ffff97f249078000 task.ti: ffff97f249078000 [ 1017.900373] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1017.911934] RSP: 0018:ffff97f24907bab0 EFLAGS: 00000246 [ 1017.918344] RAX: 0000000000000000 RBX: ffff97f2bfc00000 RCX: 0000000000810000 [ 1017.926793] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 1017.935239] RBP: ffff97f24907bab0 R08: ffff97f2bfc1b8c0 R09: 0000000000000000 [ 1017.943686] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f24907ba78 [ 1017.952133] R13: ffffffffa00ec9b1 R14: ffff97f24907ba58 R15: ffff97f24907ba88 [ 1017.960580] FS: 0000000000000000(0000) GS:ffff97f2bfc00000(0000) knlGS:0000000000000000 [ 1017.970095] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1017.976991] CR2: 00007ffff7ad2d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1017.985437] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1017.993886] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1018.002334] Call Trace: [ 1018.005546] [] queued_spin_lock_slowpath+0xb/0xf [ 1018.013024] [] _raw_spin_lock+0x30/0x40 [ 1018.019636] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1018.027314] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1018.035283] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1018.043829] [] ? wake_up_atomic_t+0x30/0x30 [ 1018.050829] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1018.058234] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1018.066511] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1018.074500] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1018.083164] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1018.091052] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1018.098366] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1018.106551] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1018.115484] [] ? __getnstimeofday64+0x3f/0xd0 [ 1018.122696] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1018.131748] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1018.139810] [] ? __wake_up_common_lock+0x91/0xc0 [ 1018.147289] [] ? sched_feat_set+0xf0/0xf0 [ 1018.154113] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1018.161591] [] ? __switch_to+0xce/0x5a0 [ 1018.168219] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1018.176959] [] kthread+0xd1/0xe0 [ 1018.182887] [] ? insert_kthread_work+0x40/0x40 [ 1018.190172] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1018.197942] [] ? insert_kthread_work+0x40/0x40 [ 1018.205226] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 1020.240523] Lustre: 7373:0:(client.c:2169:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1644342570/real 0] req@ffff97f248055580 x1724217557346816/t0(0) o400->ls1-OST0003-osc-MDT0000@172.19.3.117@o2ib600:28/4 lens 224/224 e 0 to 1 dl 1644342676 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [ 1020.272059] Lustre: 7373:0:(client.c:2169:ptlrpc_expire_one_request()) Skipped 84 previous similar messages [ 1020.623597] LustreError: 8580:0:(ldlm_lib.c:3348:target_bulk_io()) @@@ timeout on bulk READ after 100+1644341669s req@ffff98321091f850 x1723716750182144/t0(0) o256->eaf434ca-7503-622a-de02-6eccbe723f5d@192.168.113.64@o2ib15:262/0 lens 304/240 e 2 to 0 dl 1644342697 ref 1 fl Interpret:/0/0 rc 0/0 [ 1021.072600] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [mdt00_008:17320] [ 1021.086601] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ll_mgs_0009:8655] [ 1021.081628] Modules linked in: osp(OE) mdd(OE) [ 1021.086602] Modules linked in: [ 1021.086603] osp(OE) [ 1021.086603] mdd(OE) [ 1021.086604] lod(OE) [ 1021.086604] mdt(OE) [ 1021.086604] lfsck(OE) [ 1021.086605] mgs(OE) [ 1021.086605] mgc(OE) [ 1021.086605] osd_zfs(OE) [ 1021.086605] lquota(OE) [ 1021.086606] fid(OE) [ 1021.086606] fld(OE) [ 1021.086606] ptlrpc(OE) [ 1021.086607] obdclass(OE) [ 1021.086607] ko2iblnd(OE) [ 1021.086607] lnet(OE) [ 1021.086607] libcfs(OE) [ 1021.086608] iTCO_wdt [ 1021.086608] iTCO_vendor_support [ 1021.086608] sb_edac [ 1021.086609] intel_powerclamp [ 1021.086609] coretemp [ 1021.086609] intel_rapl [ 1021.086610] iosf_mbi [ 1021.086610] rpcrdma [ 1021.086610] ib_iser [ 1021.086610] kvm [ 1021.086611] irqbypass [ 1021.086611] joydev [ 1021.086611] pcspkr [ 1021.086611] lpc_ich [ 1021.086612] i2c_i801 [ 1021.086612] ioatdma [ 1021.086612] ses [ 1021.086612] enclosure [ 1021.086613] sg [ 1021.086613] ipmi_si [ 1021.086613] ipmi_devintf [ 1021.086614] ipmi_msghandler [ 1021.086614] acpi_power_meter [ 1021.086614] dm_round_robin [ 1021.086614] acpi_cpufreq [ 1021.086615] ib_ipoib [ 1021.086615] rdma_ucm [ 1021.086615] ib_umad [ 1021.086616] sch_fq_codel [ 1021.086616] iw_cxgb4 [ 1021.086616] rdma_cm [ 1021.086616] iw_cm [ 1021.086617] ib_cm [ 1021.086617] iw_cxgb3 [ 1021.086617] zfs(POE) [ 1021.086618] binfmt_misc [ 1021.086618] zunicode(POE) [ 1021.086618] zavl(POE) [ 1021.086619] icp(POE) [ 1021.086619] zcommon(POE) [ 1021.086619] znvpair(POE) [ 1021.086619] spl(OE) [ 1021.086620] msr_safe(OE) [ 1021.086620] ip_tables [ 1021.086620] nfsv3 [ 1021.086621] nfs_acl [ 1021.086621] rpcsec_gss_krb5 [ 1021.086621] auth_rpcgss [ 1021.086622] nfsv4 [ 1021.086622] dns_resolver [ 1021.086622] nfs [ 1021.086623] lockd [ 1021.086623] grace [ 1021.086623] fscache [ 1021.086623] overlay(T) [ 1021.086624] ext4 [ 1021.086624] mbcache [ 1021.086624] jbd2 [ 1021.086625] dm_service_time [ 1021.086625] sd_mod [ 1021.086625] crc_t10dif [ 1021.086626] crct10dif_generic [ 1021.086626] mlx5_ib [ 1021.086626] ib_uverbs [ 1021.086626] ib_core [ 1021.086627] be2iscsi [ 1021.086627] bnx2i [ 1021.086627] cnic [ 1021.086628] uio [ 1021.086628] cxgb4i [ 1021.086628] cxgb4 [ 1021.086628] cxgb3i [ 1021.086629] cxgb3 [ 1021.086629] mdio [ 1021.086629] libcxgbi [ 1021.086630] libcxgb [ 1021.086630] qla4xxx [ 1021.086630] iscsi_boot_sysfs [ 1021.086630] mgag200 [ 1021.086631] i2c_algo_bit [ 1021.086631] 8021q [ 1021.086631] drm_kms_helper [ 1021.086632] garp [ 1021.086632] syscopyarea [ 1021.086632] mrp [ 1021.086633] crct10dif_pclmul [ 1021.086633] stp [ 1021.086633] sysfillrect [ 1021.086633] crct10dif_common [ 1021.086634] sysimgblt [ 1021.086634] crc32_pclmul [ 1021.086634] llc [ 1021.086635] fb_sys_fops [ 1021.086635] crc32c_intel [ 1021.086635] ttm [ 1021.086636] ghash_clmulni_intel [ 1021.086636] mlx5_core [ 1021.086636] drm [ 1021.086637] mxm_wmi [ 1021.086637] ahci [ 1021.086637] ixgbe(OE) [ 1021.086638] aesni_intel [ 1021.086638] mpt3sas [ 1021.086638] mlxfw [ 1021.086639] dm_multipath [ 1021.086639] lrw [ 1021.086639] devlink [ 1021.086639] libahci [ 1021.086640] gf128mul [ 1021.086640] dca [ 1021.086640] glue_helper [ 1021.086641] ablk_helper [ 1021.086641] raid_class [ 1021.086641] ptp [ 1021.086642] drm_panel_orientation_quirks [ 1021.086642] libata [ 1021.086642] cryptd [ 1021.086643] scsi_transport_sas [ 1021.086643] pps_core [ 1021.086643] wmi [ 1021.086643] sunrpc [ 1021.086644] dm_mirror [ 1021.086644] dm_region_hash [ 1021.086644] dm_log [ 1021.086645] dm_mod [ 1021.086645] iscsi_tcp [ 1021.086645] libiscsi_tcp [ 1021.086645] libiscsi [ 1021.086646] scsi_transport_iscsi [ 1021.086646] fuse [ 1021.086646] [ 1021.086648] CPU: 3 PID: 8655 Comm: ll_mgs_0009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1021.086648] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1021.086649] task: ffff983229036300 ti: ffff98322904c000 task.ti: ffff98322904c000 [ 1021.086650] RIP: 0010:[] [ 1021.086652] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 1021.086652] RSP: 0018:ffff98322904fa50 EFLAGS: 00000246 [ 1021.086653] RAX: 0000000000000000 RBX: ffff97f2bfac0000 RCX: 0000000000190000 [ 1021.086654] RDX: ffff9832bf39b8c0 RSI: 0000000000710000 RDI: ffff97f2a83bcf40 [ 1021.086654] RBP: ffff98322904fa50 R08: ffff97f2bfadb8c0 R09: 0000000000000000 [ 1021.086655] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98322904fa18 [ 1021.086655] R13: ffffffffa00ec9b1 R14: ffff98322904f9f8 R15: ffff98322904fa28 [ 1021.086656] FS: 0000000000000000(0000) GS:ffff97f2bfac0000(0000) knlGS:0000000000000000 [ 1021.086657] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1021.086658] CR2: 00007ffff579f624 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 1021.086658] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1021.086659] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1021.086659] Call Trace: [ 1021.086662] [] queued_spin_lock_slowpath+0xb/0xf [ 1021.086663] [] _raw_spin_lock+0x30/0x40 [ 1021.086668] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1021.086673] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1021.086679] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1021.086681] [] ? wake_up_atomic_t+0x30/0x30 [ 1021.086687] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1021.086706] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1021.086722] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1021.086729] [] ? lnet_ptl_attach_md+0x38e/0x540 [lnet] [ 1021.086730] [] ? enqueue_task_fair+0x208/0x6c0 [ 1021.086732] [] ? sched_clock_cpu+0xa5/0xe0 [ 1021.086733] [] ? check_preempt_curr+0x80/0xa0 [ 1021.086735] [] ? ttwu_do_wakeup+0x19/0x100 [ 1021.086758] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1021.086779] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1021.086780] [] ? __getnstimeofday64+0x3f/0xd0 [ 1021.086800] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1021.086819] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1021.086820] [] ? __wake_up_common_lock+0x91/0xc0 [ 1021.086821] [] ? sched_feat_set+0xf0/0xf0 [ 1021.086839] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1021.086841] [] ? __switch_to+0xce/0x5a0 [ 1021.086859] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1021.086861] [] kthread+0xd1/0xe0 [ 1021.086862] [] ? insert_kthread_work+0x40/0x40 [ 1021.086864] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1021.086865] [] ? insert_kthread_work+0x40/0x40 [ 1021.086865] Code: [ 1021.086866] c1 [ 1021.086866] e8 [ 1021.086866] 13 [ 1021.086867] 48 [ 1021.086867] c1 [ 1021.086867] ea [ 1021.086867] 0d [ 1021.086868] 48 [ 1021.086868] 98 [ 1021.086868] 83 [ 1021.086868] e2 [ 1021.086869] 30 [ 1021.086869] 48 [ 1021.086869] 81 [ 1021.086869] c2 [ 1021.086870] c0 [ 1021.086870] b8 [ 1021.086870] 01 [ 1021.086871] 00 [ 1021.086871] 48 [ 1021.086871] 03 [ 1021.086871] 14 [ 1021.086872] c5 [ 1021.086872] e0 [ 1021.086872] 17 [ 1021.086872] d5 [ 1021.086873] a0 [ 1021.086873] 4c [ 1021.086873] 89 [ 1021.086873] 02 [ 1021.086874] 41 [ 1021.086874] 8b [ 1021.086874] 40 [ 1021.086874] 08 [ 1021.086875] 85 [ 1021.086875] c0 [ 1021.086875] 75 [ 1021.086875] 0f [ 1021.086876] 0f [ 1021.086876] 1f [ 1021.086876] 44 [ 1021.086876] 00 [ 1021.086877] 00 [ 1021.086877] [ 1021.086877] 90 [ 1021.086878] 41 [ 1021.086878] 8b [ 1021.086878] 40 [ 1021.086878] 08 [ 1021.086879] 85 [ 1021.086879] c0 [ 1021.086879] 74 [ 1021.086879] f6 [ 1021.086880] 4d [ 1021.086880] 8b [ 1021.086880] 08 [ 1021.086880] 4d [ 1021.086881] 85 [ 1021.086881] c9 [ 1021.086881] 74 [ 1021.086881] 04 [ 1021.086882] 41 [ 1021.086882] 0f [ 1021.086882] 18 [ 1021.086882] [ 1021.094602] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [mdt00_002:10663] [ 1021.094602] Modules linked in: [ 1021.094603] osp(OE) [ 1021.094604] mdd(OE) [ 1021.094604] lod(OE) [ 1021.094604] mdt(OE) [ 1021.094605] lfsck(OE) [ 1021.094605] mgs(OE) [ 1021.094606] mgc(OE) [ 1021.094606] osd_zfs(OE) [ 1021.094607] lquota(OE) [ 1021.094607] fid(OE) [ 1021.094607] fld(OE) [ 1021.094608] ptlrpc(OE) [ 1021.094608] obdclass(OE) [ 1021.094609] ko2iblnd(OE) [ 1021.094609] lnet(OE) [ 1021.094610] libcfs(OE) [ 1021.094610] iTCO_wdt [ 1021.094610] iTCO_vendor_support [ 1021.094611] sb_edac [ 1021.094611] intel_powerclamp [ 1021.094612] coretemp [ 1021.094612] intel_rapl [ 1021.094612] iosf_mbi [ 1021.094613] rpcrdma [ 1021.094613] ib_iser [ 1021.094613] kvm [ 1021.094614] irqbypass [ 1021.094614] joydev [ 1021.094615] pcspkr [ 1021.094615] lpc_ich [ 1021.094615] i2c_i801 [ 1021.094616] ioatdma [ 1021.094616] ses [ 1021.094617] enclosure [ 1021.094617] sg [ 1021.094617] ipmi_si [ 1021.094618] ipmi_devintf [ 1021.094618] ipmi_msghandler [ 1021.094619] acpi_power_meter [ 1021.094619] dm_round_robin [ 1021.094619] acpi_cpufreq [ 1021.094620] ib_ipoib [ 1021.094620] rdma_ucm [ 1021.094620] ib_umad [ 1021.094621] sch_fq_codel [ 1021.094621] iw_cxgb4 [ 1021.094622] rdma_cm [ 1021.094622] iw_cm [ 1021.094622] ib_cm [ 1021.094623] iw_cxgb3 [ 1021.094623] zfs(POE) [ 1021.094624] binfmt_misc [ 1021.094624] zunicode(POE) [ 1021.094625] zavl(POE) [ 1021.094625] icp(POE) [ 1021.094626] zcommon(POE) [ 1021.094626] znvpair(POE) [ 1021.094627] spl(OE) [ 1021.094627] msr_safe(OE) [ 1021.094628] ip_tables [ 1021.094628] nfsv3 [ 1021.094629] nfs_acl [ 1021.094629] rpcsec_gss_krb5 [ 1021.094630] auth_rpcgss [ 1021.094630] nfsv4 [ 1021.094630] dns_resolver [ 1021.094631] nfs [ 1021.094631] lockd [ 1021.094632] grace [ 1021.094632] fscache [ 1021.094632] overlay(T) [ 1021.094633] ext4 [ 1021.094633] mbcache [ 1021.094634] jbd2 [ 1021.094634] dm_service_time [ 1021.094635] sd_mod [ 1021.094635] crc_t10dif [ 1021.094635] crct10dif_generic [ 1021.094636] mlx5_ib [ 1021.094636] ib_uverbs [ 1021.094636] ib_core [ 1021.094637] be2iscsi [ 1021.094637] bnx2i [ 1021.094638] cnic [ 1021.094638] uio [ 1021.094639] cxgb4i [ 1021.094639] cxgb4 [ 1021.094639] cxgb3i [ 1021.094640] cxgb3 [ 1021.094640] mdio [ 1021.094641] libcxgbi [ 1021.094641] libcxgb [ 1021.094641] qla4xxx [ 1021.094642] iscsi_boot_sysfs [ 1021.094642] mgag200 [ 1021.094643] i2c_algo_bit [ 1021.094643] 8021q [ 1021.094643] drm_kms_helper [ 1021.094644] garp [ 1021.094644] syscopyarea [ 1021.094645] mrp [ 1021.094645] crct10dif_pclmul [ 1021.094645] stp [ 1021.094646] sysfillrect [ 1021.094646] crct10dif_common [ 1021.094647] sysimgblt [ 1021.094647] crc32_pclmul [ 1021.094647] llc [ 1021.094648] fb_sys_fops [ 1021.094648] crc32c_intel [ 1021.094649] ttm [ 1021.094649] ghash_clmulni_intel [ 1021.094649] mlx5_core [ 1021.094650] drm [ 1021.094650] mxm_wmi [ 1021.094650] ahci [ 1021.094651] ixgbe(OE) [ 1021.094651] aesni_intel [ 1021.094652] mpt3sas [ 1021.094652] mlxfw [ 1021.094653] dm_multipath [ 1021.094653] lrw [ 1021.094653] devlink [ 1021.094654] libahci [ 1021.094654] gf128mul [ 1021.094655] dca [ 1021.094655] glue_helper [ 1021.094655] ablk_helper [ 1021.094656] raid_class [ 1021.094656] ptp [ 1021.094657] drm_panel_orientation_quirks [ 1021.094657] libata [ 1021.094657] cryptd [ 1021.094658] scsi_transport_sas [ 1021.094658] pps_core [ 1021.094659] wmi [ 1021.094659] sunrpc [ 1021.094660] dm_mirror [ 1021.094660] dm_region_hash [ 1021.094660] dm_log [ 1021.094661] dm_mod [ 1021.094661] iscsi_tcp [ 1021.094662] libiscsi_tcp [ 1021.094662] libiscsi [ 1021.094662] scsi_transport_iscsi [ 1021.094663] fuse [ 1021.094663] [ 1021.094665] CPU: 5 PID: 10663 Comm: mdt00_002 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1021.094666] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1021.094667] task: ffff97f28e72a100 ti: ffff97f28e600000 task.ti: ffff97f28e600000 [ 1021.094668] RIP: 0010:[] [ 1021.094670] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1021.094671] RSP: 0018:ffff97f28e603ab0 EFLAGS: 00000246 [ 1021.094672] RAX: 0000000000000000 RBX: ffff97f2bfb40000 RCX: 0000000000290000 [ 1021.094673] RDX: ffff97f2bfc1b8c0 RSI: 0000000000810000 RDI: ffff97f2a83bcf40 [ 1021.094674] RBP: ffff97f28e603ab0 R08: ffff97f2bfb5b8c0 R09: 0000000000000000 [ 1021.094675] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f28e603a78 [ 1021.094676] R13: ffffffffa00ec9b1 R14: ffff97f28e603a58 R15: ffff97f28e603a88 [ 1021.094677] FS: 0000000000000000(0000) GS:ffff97f2bfb40000(0000) knlGS:0000000000000000 [ 1021.094678] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1021.094679] CR2: 00007ffff1096160 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 1021.094680] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1021.094681] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1021.094681] Call Trace: [ 1021.094684] [] queued_spin_lock_slowpath+0xb/0xf [ 1021.094686] [] _raw_spin_lock+0x30/0x40 [ 1021.094693] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1021.094699] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1021.094707] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1021.094709] [] ? wake_up_atomic_t+0x30/0x30 [ 1021.094717] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1021.094742] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1021.094766] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1021.094792] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1021.094816] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1021.094839] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1021.094869] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1021.094897] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1021.094899] [] ? __getnstimeofday64+0x3f/0xd0 [ 1021.094924] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1021.094949] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1021.094950] [] ? __wake_up_common_lock+0x91/0xc0 [ 1021.094952] [] ? sched_feat_set+0xf0/0xf0 [ 1021.094976] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1021.094978] [] ? __switch_to+0xce/0x5a0 [ 1021.095002] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1021.095004] [] kthread+0xd1/0xe0 [ 1021.095006] [] ? insert_kthread_work+0x40/0x40 [ 1021.095007] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1021.095009] [] ? insert_kthread_work+0x40/0x40 [ 1021.095010] Code: [ 1021.095010] 0d [ 1021.095011] 48 [ 1021.095011] 98 [ 1021.095012] 83 [ 1021.095012] e2 [ 1021.095012] 30 [ 1021.095013] 48 [ 1021.095013] 81 [ 1021.095013] c2 [ 1021.095014] c0 [ 1021.095014] b8 [ 1021.095015] 01 [ 1021.095015] 00 [ 1021.095015] 48 [ 1021.095016] 03 [ 1021.095016] 14 [ 1021.095016] c5 [ 1021.095017] e0 [ 1021.095017] 17 [ 1021.095018] d5 [ 1021.095018] a0 [ 1021.095018] 4c [ 1021.095019] 89 [ 1021.095019] 02 [ 1021.095019] 41 [ 1021.095020] 8b [ 1021.095020] 40 [ 1021.095020] 08 [ 1021.095021] 85 [ 1021.095021] c0 [ 1021.095021] 75 [ 1021.095022] 0f [ 1021.095022] 0f [ 1021.095023] 1f [ 1021.095023] 44 [ 1021.095023] 00 [ 1021.095024] 00 [ 1021.095024] f3 [ 1021.095024] 90 [ 1021.095025] 41 [ 1021.095025] 8b [ 1021.095025] 40 [ 1021.095026] 08 [ 1021.095026] <85> [ 1021.095026] c0 [ 1021.095027] 74 [ 1021.095027] f6 [ 1021.095028] 4d [ 1021.095028] 8b [ 1021.095028] 08 [ 1021.095029] 4d [ 1021.095029] 85 [ 1021.095029] c9 [ 1021.095030] 74 [ 1021.095030] 04 [ 1021.095030] 41 [ 1021.095031] 0f [ 1021.095031] 18 [ 1021.095031] 09 [ 1021.095032] 8b [ 1021.095032] 17 [ 1021.095032] 0f [ 1021.095033] b7 [ 1021.095033] c2 [ 1021.095033] [ 1021.201617] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [mdt01_003:10825] [ 1021.201617] Modules linked in: [ 1021.201618] osp(OE) [ 1021.201619] mdd(OE) [ 1021.201619] lod(OE) [ 1021.201620] mdt(OE) [ 1021.201621] lfsck(OE) [ 1021.201621] mgs(OE) [ 1021.201622] mgc(OE) [ 1021.201622] osd_zfs(OE) [ 1021.201623] lquota(OE) [ 1021.201623] fid(OE) [ 1021.201624] fld(OE) [ 1021.201624] ptlrpc(OE) [ 1021.201625] obdclass(OE) [ 1021.201625] ko2iblnd(OE) [ 1021.201626] lnet(OE) [ 1021.201626] libcfs(OE) [ 1021.201627] iTCO_wdt [ 1021.201628] iTCO_vendor_support [ 1021.201628] sb_edac [ 1021.201629] intel_powerclamp [ 1021.201629] coretemp [ 1021.201630] intel_rapl [ 1021.201630] iosf_mbi [ 1021.201631] rpcrdma [ 1021.201631] ib_iser [ 1021.201631] kvm [ 1021.201632] irqbypass [ 1021.201632] joydev [ 1021.201633] pcspkr [ 1021.201633] lpc_ich [ 1021.201634] i2c_i801 [ 1021.201635] ioatdma [ 1021.201635] ses [ 1021.201635] enclosure [ 1021.201636] sg [ 1021.201636] ipmi_si [ 1021.201637] ipmi_devintf [ 1021.201637] ipmi_msghandler [ 1021.201638] acpi_power_meter [ 1021.201638] dm_round_robin [ 1021.201639] acpi_cpufreq [ 1021.201639] ib_ipoib [ 1021.201640] rdma_ucm [ 1021.201640] ib_umad [ 1021.201641] sch_fq_codel [ 1021.201641] iw_cxgb4 [ 1021.201642] rdma_cm [ 1021.201642] iw_cm [ 1021.201643] ib_cm [ 1021.201643] iw_cxgb3 [ 1021.201644] zfs(POE) [ 1021.201645] binfmt_misc [ 1021.201645] zunicode(POE) [ 1021.201646] zavl(POE) [ 1021.201646] icp(POE) [ 1021.201647] zcommon(POE) [ 1021.201647] znvpair(POE) [ 1021.201648] spl(OE) [ 1021.201648] msr_safe(OE) [ 1021.201649] ip_tables [ 1021.201650] nfsv3 [ 1021.201650] nfs_acl [ 1021.201651] rpcsec_gss_krb5 [ 1021.201651] auth_rpcgss [ 1021.201652] nfsv4 [ 1021.201652] dns_resolver [ 1021.201653] nfs [ 1021.201653] lockd [ 1021.201654] grace [ 1021.201654] fscache [ 1021.201655] overlay(T) [ 1021.201655] ext4 [ 1021.201656] mbcache [ 1021.201656] jbd2 [ 1021.201657] dm_service_time [ 1021.201657] sd_mod [ 1021.201658] crc_t10dif [ 1021.201658] crct10dif_generic [ 1021.201659] mlx5_ib [ 1021.201659] ib_uverbs [ 1021.201660] ib_core [ 1021.201660] be2iscsi [ 1021.201661] bnx2i [ 1021.201661] cnic [ 1021.201662] uio [ 1021.201662] cxgb4i [ 1021.201663] cxgb4 [ 1021.201663] cxgb3i [ 1021.201664] cxgb3 [ 1021.201664] mdio [ 1021.201665] libcxgbi [ 1021.201665] libcxgb [ 1021.201666] qla4xxx [ 1021.201666] iscsi_boot_sysfs [ 1021.201667] mgag200 [ 1021.201668] i2c_algo_bit [ 1021.201668] 8021q [ 1021.201669] drm_kms_helper [ 1021.201669] garp [ 1021.201670] syscopyarea [ 1021.201671] mrp [ 1021.201671] crct10dif_pclmul [ 1021.201671] stp [ 1021.201672] sysfillrect [ 1021.201672] crct10dif_common [ 1021.201673] sysimgblt [ 1021.201673] crc32_pclmul [ 1021.201674] llc [ 1021.201674] fb_sys_fops [ 1021.201675] crc32c_intel [ 1021.201675] ttm [ 1021.201677] ghash_clmulni_intel [ 1021.201677] mlx5_core [ 1021.201678] drm [ 1021.201678] mxm_wmi [ 1021.201679] ahci [ 1021.201679] ixgbe(OE) [ 1021.201680] aesni_intel [ 1021.201680] mpt3sas [ 1021.201681] mlxfw [ 1021.201681] dm_multipath [ 1021.201682] lrw [ 1021.201682] devlink [ 1021.201683] libahci [ 1021.201683] gf128mul [ 1021.201684] dca [ 1021.201684] glue_helper [ 1021.201685] ablk_helper [ 1021.201686] raid_class [ 1021.201686] ptp [ 1021.201687] drm_panel_orientation_quirks [ 1021.201687] libata [ 1021.201688] cryptd [ 1021.201688] scsi_transport_sas [ 1021.201689] pps_core [ 1021.201689] wmi [ 1021.201690] sunrpc [ 1021.201690] dm_mirror [ 1021.201691] dm_region_hash [ 1021.201691] dm_log [ 1021.201692] dm_mod [ 1021.201692] iscsi_tcp [ 1021.201692] libiscsi_tcp [ 1021.201693] libiscsi [ 1021.201693] scsi_transport_iscsi [ 1021.201694] fuse [ 1021.201694] [ 1021.201697] CPU: 13 PID: 10825 Comm: mdt01_003 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1021.201698] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1021.201699] task: ffff98322385d280 ti: ffff983225500000 task.ti: ffff983225500000 [ 1021.201700] RIP: 0010:[] [ 1021.201707] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1021.201708] RSP: 0018:ffff983225503a50 EFLAGS: 00000246 [ 1021.201709] RAX: 0000000000000000 RBX: ffff9832bf340000 RCX: 0000000000690000 [ 1021.201710] RDX: ffff97f2bfddb8c0 RSI: 0000000000b90000 RDI: ffff97f2a83bcf40 [ 1021.201711] RBP: ffff983225503a50 R08: ffff9832bf35b8c0 R09: 0000000000000000 [ 1021.201712] R10: 0000000000000001 R11: 0000000000000005 R12: ffff983225503a18 [ 1021.201713] R13: ffffffffa00ec9b1 R14: ffff9832255039f8 R15: ffff983225503a28 [ 1021.201714] FS: 0000000000000000(0000) GS:ffff9832bf340000(0000) knlGS:0000000000000000 [ 1021.201715] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1021.201716] CR2: 00002aaaab176a00 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 1021.201718] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1021.201719] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1021.201719] Call Trace: [ 1021.201725] [] queued_spin_lock_slowpath+0xb/0xf [ 1021.201729] [] _raw_spin_lock+0x30/0x40 [ 1021.201742] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1021.201765] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1021.201768] [] ? wake_up_atomic_t+0x30/0x30 [ 1021.201776] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1021.201816] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1021.201840] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1021.201844] [] ? enqueue_task_fair+0x208/0x6c0 [ 1021.201846] [] ? check_preempt_curr+0x80/0xa0 [ 1021.201847] [] ? ttwu_do_wakeup+0x19/0x100 [ 1021.201885] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1021.201913] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1021.201916] [] ? __getnstimeofday64+0x3f/0xd0 [ 1021.201941] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1021.201966] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1021.201967] [] ? __wake_up_common_lock+0x91/0xc0 [ 1021.201969] [] ? sched_feat_set+0xf0/0xf0 [ 1021.201994] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1021.202018] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1021.202020] [] kthread+0xd1/0xe0 [ 1021.202022] [] ? insert_kthread_work+0x40/0x40 [ 1021.202024] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1021.202026] [] ? insert_kthread_work+0x40/0x40 [ 1021.202027] Code: [ 1021.202027] 0d [ 1021.202028] 48 [ 1021.202028] 98 [ 1021.202028] 83 [ 1021.202029] e2 [ 1021.202029] 30 [ 1021.202029] 48 [ 1021.202030] 81 [ 1021.202030] c2 [ 1021.202031] c0 [ 1021.202031] b8 [ 1021.202031] 01 [ 1021.202032] 00 [ 1021.202032] 48 [ 1021.202032] 03 [ 1021.202033] 14 [ 1021.202033] c5 [ 1021.202033] e0 [ 1021.202034] 17 [ 1021.202034] d5 [ 1021.202034] a0 [ 1021.202035] 4c [ 1021.202035] 89 [ 1021.202035] 02 [ 1021.202036] 41 [ 1021.202036] 8b [ 1021.202036] 40 [ 1021.202037] 08 [ 1021.202037] 85 [ 1021.202037] c0 [ 1021.202038] 75 [ 1021.202038] 0f [ 1021.202038] 0f [ 1021.202039] 1f [ 1021.202039] 44 [ 1021.202039] 00 [ 1021.202040] 00 [ 1021.202040] f3 [ 1021.202040] 90 [ 1021.202041] 41 [ 1021.202041] 8b [ 1021.202041] 40 [ 1021.202042] 08 [ 1021.202042] <85> [ 1021.202043] c0 [ 1021.202043] 74 [ 1021.202043] f6 [ 1021.202044] 4d [ 1021.202044] 8b [ 1021.202045] 08 [ 1021.202045] 4d [ 1021.202046] 85 [ 1021.202046] c9 [ 1021.202046] 74 [ 1021.202047] 04 [ 1021.202047] 41 [ 1021.202047] 0f [ 1021.202048] 18 [ 1021.202048] 09 [ 1021.202048] 8b [ 1021.202049] 17 [ 1021.202049] 0f [ 1021.202049] b7 [ 1021.202050] c2 [ 1021.202050] [ 1022.619713] lod(OE) [ 1022.622643] mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 [ 1022.700817] cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1022.757435] CPU: 0 PID: 17320 Comm: mdt00_008 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1022.771702] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1022.784027] task: ffff98321989e300 ti: ffff98321cd1c000 task.ti: ffff98321cd1c000 [ 1022.792863] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1022.804424] RSP: 0018:ffff98321cd1fab0 EFLAGS: 00000246 [ 1022.810836] RAX: 0000000000000000 RBX: ffff97f2bfa00000 RCX: 0000000000010000 [ 1022.819284] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 1022.827731] RBP: ffff98321cd1fab0 R08: ffff97f2bfa1b8c0 R09: 0000000000000000 [ 1022.836178] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321cd1fa78 [ 1022.844625] R13: ffffffffa00ec9b1 R14: ffff98321cd1fa58 R15: ffff98321cd1fa88 [ 1022.853074] FS: 0000000000000000(0000) GS:ffff97f2bfa00000(0000) knlGS:0000000000000000 [ 1022.862589] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1022.869485] CR2: 00007ffff7ff8000 CR3: 0000007f0e586000 CR4: 00000000003607f0 [ 1022.877932] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1022.886380] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1022.894826] Call Trace: [ 1022.898039] [] queued_spin_lock_slowpath+0xb/0xf [ 1022.905516] [] _raw_spin_lock+0x30/0x40 [ 1022.912127] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1022.919802] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1022.927773] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1022.936319] [] ? wake_up_atomic_t+0x30/0x30 [ 1022.943319] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1022.950725] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1022.959001] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1022.966964] [] ? check_preempt_curr+0x80/0xa0 [ 1022.974175] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1022.982064] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1022.989379] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1022.997565] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1023.006498] [] ? __getnstimeofday64+0x3f/0xd0 [ 1023.013711] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1023.022766] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1023.030828] [] ? __wake_up_common_lock+0x91/0xc0 [ 1023.038306] [] ? sched_feat_set+0xf0/0xf0 [ 1023.045130] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1023.052611] [] ? __switch_to+0xce/0x5a0 [ 1023.059242] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1023.067982] [] kthread+0xd1/0xe0 [ 1023.073910] [] ? insert_kthread_work+0x40/0x40 [ 1023.081196] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1023.088967] [] ? insert_kthread_work+0x40/0x40 [ 1023.096250] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1023.989272] Lustre: 10831:0:(service.c:2169:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (61:149s); client may timeout. req@ffff9832238a5050 x1724217779348480/t0(0) o101->ls1-MDT0009-mdtlov_UUID@172.19.3.107@o2ib600:96/0 lens 328/312 e 0 to 0 dl 1644342531 ref 1 fl Complete:/0/0 rc 0/0 [ 1024.020628] LNet: Service thread pid 10831 completed after 209.69s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [ 1024.039156] LNet: Skipped 1 previous similar message [ 1029.229539] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [mdt00_009:17321] [ 1029.238667] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1029.318484] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1029.379535] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1029.393896] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1029.406222] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 1029.415056] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1029.426618] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 1029.433030] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 1029.441476] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 1029.449924] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 1029.458371] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 1029.466816] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 1029.475264] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 1029.484777] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1029.491673] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1029.500119] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1029.508566] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1029.517013] Call Trace: [ 1029.520227] [] queued_spin_lock_slowpath+0xb/0xf [ 1029.527704] [] _raw_spin_lock+0x30/0x40 [ 1029.534314] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1029.541983] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1029.550430] [] ? wake_up_atomic_t+0x30/0x30 [ 1029.557429] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1029.564831] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1029.573099] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1029.581660] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1029.589816] [] ? enqueue_task_fair+0x208/0x6c0 [ 1029.597100] [] ? sched_clock_cpu+0xa5/0xe0 [ 1029.603996] [] ? check_preempt_curr+0x80/0xa0 [ 1029.611183] [] ? ttwu_do_wakeup+0x19/0x100 [ 1029.618103] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1029.626280] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1029.635214] [] ? __getnstimeofday64+0x3f/0xd0 [ 1029.642420] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1029.651467] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1029.659528] [] ? __wake_up_common_lock+0x91/0xc0 [ 1029.667006] [] ? sched_feat_set+0xf0/0xf0 [ 1029.673829] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1029.681308] [] ? __switch_to+0xce/0x5a0 [ 1029.687938] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1029.696678] [] kthread+0xd1/0xe0 [ 1029.702605] [] ? insert_kthread_work+0x40/0x40 [ 1029.709889] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1029.717659] [] ? insert_kthread_work+0x40/0x40 [ 1029.724944] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1033.083984] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [mdt00_013:17594] [ 1033.093016] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1033.180996] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [mdt01_022:17601] [ 1033.172842] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx [ 1033.180997] Modules linked in: [ 1033.180998] osp(OE) [ 1033.180998] mdd(OE) [ 1033.180999] lod(OE) [ 1033.180999] mdt(OE) [ 1033.180999] lfsck(OE) [ 1033.181000] mgs(OE) [ 1033.181000] mgc(OE) [ 1033.181001] osd_zfs(OE) [ 1033.181001] lquota(OE) [ 1033.181001] fid(OE) [ 1033.181002] fld(OE) [ 1033.181002] ptlrpc(OE) [ 1033.181002] obdclass(OE) [ 1033.181003] ko2iblnd(OE) [ 1033.181003] lnet(OE) [ 1033.181003] libcfs(OE) [ 1033.181004] iTCO_wdt [ 1033.181005] iTCO_vendor_support [ 1033.181005] sb_edac [ 1033.181006] intel_powerclamp [ 1033.181006] coretemp [ 1033.181007] intel_rapl [ 1033.181007] iosf_mbi [ 1033.181008] rpcrdma [ 1033.181008] ib_iser [ 1033.181008] kvm [ 1033.181009] irqbypass [ 1033.181010] joydev [ 1033.181010] pcspkr [ 1033.181010] lpc_ich [ 1033.181011] i2c_i801 [ 1033.181011] ioatdma [ 1033.181012] ses [ 1033.181012] enclosure [ 1033.181013] sg [ 1033.181013] ipmi_si [ 1033.181014] ipmi_devintf [ 1033.181014] ipmi_msghandler [ 1033.181015] acpi_power_meter [ 1033.181015] dm_round_robin [ 1033.181016] acpi_cpufreq [ 1033.181016] ib_ipoib [ 1033.181017] rdma_ucm [ 1033.181017] ib_umad [ 1033.181017] sch_fq_codel [ 1033.181018] iw_cxgb4 [ 1033.181018] rdma_cm [ 1033.181019] iw_cm [ 1033.181019] ib_cm [ 1033.181020] iw_cxgb3 [ 1033.181020] zfs(POE) [ 1033.181020] binfmt_misc [ 1033.181021] zunicode(POE) [ 1033.181021] zavl(POE) [ 1033.181022] icp(POE) [ 1033.181022] zcommon(POE) [ 1033.181023] znvpair(POE) [ 1033.181023] spl(OE) [ 1033.181024] msr_safe(OE) [ 1033.181024] ip_tables [ 1033.181025] nfsv3 [ 1033.181025] nfs_acl [ 1033.181026] rpcsec_gss_krb5 [ 1033.181026] auth_rpcgss [ 1033.181027] nfsv4 [ 1033.181027] dns_resolver [ 1033.181028] nfs [ 1033.181028] lockd [ 1033.181028] grace [ 1033.181029] fscache [ 1033.181029] overlay(T) [ 1033.181030] ext4 [ 1033.181030] mbcache [ 1033.181031] jbd2 [ 1033.181031] dm_service_time [ 1033.181032] sd_mod [ 1033.181032] crc_t10dif [ 1033.181032] crct10dif_generic [ 1033.181033] mlx5_ib [ 1033.181033] ib_uverbs [ 1033.181034] ib_core [ 1033.181034] be2iscsi [ 1033.181035] bnx2i [ 1033.181035] cnic [ 1033.181035] uio [ 1033.181036] cxgb4i [ 1033.181036] cxgb4 [ 1033.181037] cxgb3i [ 1033.181037] cxgb3 [ 1033.181038] mdio [ 1033.181038] libcxgbi [ 1033.181039] libcxgb [ 1033.181039] qla4xxx [ 1033.181040] iscsi_boot_sysfs [ 1033.181040] mgag200 [ 1033.181041] i2c_algo_bit [ 1033.181041] 8021q [ 1033.181042] drm_kms_helper [ 1033.181042] garp [ 1033.181043] syscopyarea [ 1033.181043] mrp [ 1033.181044] crct10dif_pclmul [ 1033.181044] stp [ 1033.181045] sysfillrect [ 1033.181045] crct10dif_common [ 1033.181045] sysimgblt [ 1033.181046] crc32_pclmul [ 1033.181046] llc [ 1033.181046] fb_sys_fops [ 1033.181047] crc32c_intel [ 1033.181047] ttm [ 1033.181048] ghash_clmulni_intel [ 1033.181048] mlx5_core [ 1033.181049] drm [ 1033.181049] mxm_wmi [ 1033.181050] ahci [ 1033.181050] ixgbe(OE) [ 1033.181051] aesni_intel [ 1033.181051] mpt3sas [ 1033.181052] mlxfw [ 1033.181052] dm_multipath [ 1033.181052] lrw [ 1033.181053] devlink [ 1033.181053] libahci [ 1033.181053] gf128mul [ 1033.181054] dca [ 1033.181054] glue_helper [ 1033.181055] ablk_helper [ 1033.181055] raid_class [ 1033.181055] ptp [ 1033.181056] drm_panel_orientation_quirks [ 1033.181056] libata [ 1033.181057] cryptd [ 1033.181057] scsi_transport_sas [ 1033.181058] pps_core [ 1033.181058] wmi [ 1033.181059] sunrpc [ 1033.181059] dm_mirror [ 1033.181059] dm_region_hash [ 1033.181060] dm_log [ 1033.181060] dm_mod [ 1033.181060] iscsi_tcp [ 1033.181061] libiscsi_tcp [ 1033.181061] libiscsi [ 1033.181062] scsi_transport_iscsi [ 1033.181062] fuse [ 1033.181062] [ 1033.181065] CPU: 8 PID: 17601 Comm: mdt01_022 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.181066] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.181067] task: ffff983218c53180 ti: ffff983217c64000 task.ti: ffff983217c64000 [ 1033.181067] RIP: 0010:[] [ 1033.181072] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1033.181072] RSP: 0018:ffff983217c67ab0 EFLAGS: 00000246 [ 1033.181073] RAX: 0000000000000000 RBX: ffff9832bf200000 RCX: 0000000000410000 [ 1033.181074] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 1033.181074] RBP: ffff983217c67ab0 R08: ffff9832bf21b8c0 R09: 0000000000000000 [ 1033.181075] R10: 0000000000000001 R11: ffffe2d93b1a2500 R12: ffff983217c67a78 [ 1033.181076] R13: ffffffffa00ec9b1 R14: ffff983217c67a58 R15: ffff983217c67a88 [ 1033.181077] FS: 0000000000000000(0000) GS:ffff9832bf200000(0000) knlGS:0000000000000000 [ 1033.181077] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.181078] CR2: 00007ffff7ff8000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1033.181079] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.181080] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.181080] Call Trace: [ 1033.181085] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.181088] [] _raw_spin_lock+0x30/0x40 [ 1033.181099] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.181112] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.181115] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.181120] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.181151] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.181170] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1033.181189] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1033.181206] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1033.181223] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1033.181250] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1033.181271] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.181273] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.181292] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.181311] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.181312] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.181313] [] ? sched_feat_set+0xf0/0xf0 [ 1033.181332] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.181350] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.181352] [] kthread+0xd1/0xe0 [ 1033.181353] [] ? insert_kthread_work+0x40/0x40 [ 1033.181355] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.181357] [] ? insert_kthread_work+0x40/0x40 [ 1033.181357] Code: [ 1033.181358] 13 [ 1033.181358] 48 [ 1033.181358] c1 [ 1033.181359] ea [ 1033.181359] 0d [ 1033.181359] 48 [ 1033.181359] 98 [ 1033.181360] 83 [ 1033.181360] e2 [ 1033.181360] 30 [ 1033.181361] 48 [ 1033.181361] 81 [ 1033.181361] c2 [ 1033.181361] c0 [ 1033.181362] b8 [ 1033.181362] 01 [ 1033.181362] 00 [ 1033.181363] 48 [ 1033.181363] 03 [ 1033.181363] 14 [ 1033.181364] c5 [ 1033.181364] e0 [ 1033.181364] 17 [ 1033.181364] d5 [ 1033.181365] a0 [ 1033.181365] 4c [ 1033.181365] 89 [ 1033.181365] 02 [ 1033.181366] 41 [ 1033.181366] 8b [ 1033.181366] 40 [ 1033.181367] 08 [ 1033.181367] 85 [ 1033.181367] c0 [ 1033.181367] 75 [ 1033.181368] 0f [ 1033.181368] 0f [ 1033.181368] 1f [ 1033.181368] 44 [ 1033.181369] 00 [ 1033.181369] 00 [ 1033.181369] f3 [ 1033.181369] 90 [ 1033.181370] <41> [ 1033.181370] 8b [ 1033.181370] 40 [ 1033.181371] 08 [ 1033.181371] 85 [ 1033.181371] c0 [ 1033.181371] 74 [ 1033.181372] f6 [ 1033.181372] 4d [ 1033.181372] 8b [ 1033.181372] 08 [ 1033.181372] 4d [ 1033.181373] 85 [ 1033.181373] c9 [ 1033.181373] 74 [ 1033.181373] 04 [ 1033.181374] 41 [ 1033.181374] 0f [ 1033.181374] 18 [ 1033.181374] 09 [ 1033.181375] 8b [ 1033.181375] [ 1033.207998] NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [mdt01_016:17554] [ 1033.207998] Modules linked in: [ 1033.207999] osp(OE) [ 1033.207999] mdd(OE) [ 1033.207999] lod(OE) [ 1033.208000] mdt(OE) [ 1033.208000] lfsck(OE) [ 1033.208001] mgs(OE) [ 1033.208001] mgc(OE) [ 1033.208002] osd_zfs(OE) [ 1033.208002] lquota(OE) [ 1033.208002] fid(OE) [ 1033.208003] fld(OE) [ 1033.208003] ptlrpc(OE) [ 1033.208004] obdclass(OE) [ 1033.208004] ko2iblnd(OE) [ 1033.208004] lnet(OE) [ 1033.208005] libcfs(OE) [ 1033.208005] iTCO_wdt [ 1033.208006] iTCO_vendor_support [ 1033.208006] sb_edac [ 1033.208007] intel_powerclamp [ 1033.208007] coretemp [ 1033.208007] intel_rapl [ 1033.208008] iosf_mbi [ 1033.208008] rpcrdma [ 1033.208008] ib_iser [ 1033.208009] kvm [ 1033.208009] irqbypass [ 1033.208009] joydev [ 1033.208010] pcspkr [ 1033.208010] lpc_ich [ 1033.208011] i2c_i801 [ 1033.208011] ioatdma [ 1033.208011] ses [ 1033.208012] enclosure [ 1033.208012] sg [ 1033.208012] ipmi_si [ 1033.208013] ipmi_devintf [ 1033.208013] ipmi_msghandler [ 1033.208014] acpi_power_meter [ 1033.208014] dm_round_robin [ 1033.208014] acpi_cpufreq [ 1033.208015] ib_ipoib [ 1033.208015] rdma_ucm [ 1033.208015] ib_umad [ 1033.208016] sch_fq_codel [ 1033.208016] iw_cxgb4 [ 1033.208017] rdma_cm [ 1033.208017] iw_cm [ 1033.208017] ib_cm [ 1033.208018] iw_cxgb3 [ 1033.208018] zfs(POE) [ 1033.208018] binfmt_misc [ 1033.208019] zunicode(POE) [ 1033.208019] zavl(POE) [ 1033.208020] icp(POE) [ 1033.208020] zcommon(POE) [ 1033.208021] znvpair(POE) [ 1033.208021] spl(OE) [ 1033.208022] msr_safe(OE) [ 1033.208022] ip_tables [ 1033.208022] nfsv3 [ 1033.208023] nfs_acl [ 1033.208023] rpcsec_gss_krb5 [ 1033.208023] auth_rpcgss [ 1033.208024] nfsv4 [ 1033.208024] dns_resolver [ 1033.208025] nfs [ 1033.208025] lockd [ 1033.208025] grace [ 1033.208026] fscache [ 1033.208026] overlay(T) [ 1033.208027] ext4 [ 1033.208027] mbcache [ 1033.208028] jbd2 [ 1033.208028] dm_service_time [ 1033.208028] sd_mod [ 1033.208029] crc_t10dif [ 1033.208029] crct10dif_generic [ 1033.208030] mlx5_ib [ 1033.208030] ib_uverbs [ 1033.208031] ib_core [ 1033.208031] be2iscsi [ 1033.208031] bnx2i [ 1033.208032] cnic [ 1033.208032] uio [ 1033.208032] cxgb4i [ 1033.208033] cxgb4 [ 1033.208033] cxgb3i [ 1033.208034] cxgb3 [ 1033.208034] mdio [ 1033.208034] libcxgbi [ 1033.208035] libcxgb [ 1033.208035] qla4xxx [ 1033.208036] iscsi_boot_sysfs [ 1033.208036] mgag200 [ 1033.208037] i2c_algo_bit [ 1033.208037] 8021q [ 1033.208037] drm_kms_helper [ 1033.208038] garp [ 1033.208038] syscopyarea [ 1033.208039] mrp [ 1033.208039] crct10dif_pclmul [ 1033.208039] stp [ 1033.208040] sysfillrect [ 1033.208040] crct10dif_common [ 1033.208040] sysimgblt [ 1033.208041] crc32_pclmul [ 1033.208041] llc [ 1033.208041] fb_sys_fops [ 1033.208042] crc32c_intel [ 1033.208042] ttm [ 1033.208043] ghash_clmulni_intel [ 1033.208043] mlx5_core [ 1033.208043] drm [ 1033.208044] mxm_wmi [ 1033.208044] ahci [ 1033.208045] ixgbe(OE) [ 1033.208045] aesni_intel [ 1033.208045] mpt3sas [ 1033.208046] mlxfw [ 1033.208046] dm_multipath [ 1033.208047] lrw [ 1033.208047] devlink [ 1033.208048] libahci [ 1033.208048] gf128mul [ 1033.208048] dca [ 1033.208049] glue_helper [ 1033.208049] ablk_helper [ 1033.208050] raid_class [ 1033.208050] ptp [ 1033.208050] drm_panel_orientation_quirks [ 1033.208051] libata [ 1033.208051] cryptd [ 1033.208052] scsi_transport_sas [ 1033.208052] pps_core [ 1033.208053] wmi [ 1033.208053] sunrpc [ 1033.208053] dm_mirror [ 1033.208054] dm_region_hash [ 1033.208054] dm_log [ 1033.208055] dm_mod [ 1033.208055] iscsi_tcp [ 1033.208055] libiscsi_tcp [ 1033.208056] libiscsi [ 1033.208056] scsi_transport_iscsi [ 1033.208056] fuse [ 1033.208057] [ 1033.208059] CPU: 14 PID: 17554 Comm: mdt01_016 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.208059] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.208060] task: ffff98321c985280 ti: ffff98321b544000 task.ti: ffff98321b544000 [ 1033.208061] RIP: 0010:[] [ 1033.208063] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1033.208064] RSP: 0018:ffff98321b547ab0 EFLAGS: 00000246 [ 1033.208065] RAX: 0000000000000000 RBX: ffff9832bf380000 RCX: 0000000000710000 [ 1033.208066] RDX: ffff9832bf4db8c0 RSI: 0000000000d90000 RDI: ffff97f2a83bcf40 [ 1033.208067] RBP: ffff98321b547ab0 R08: ffff9832bf39b8c0 R09: 0000000000000000 [ 1033.208068] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321b547a78 [ 1033.208068] R13: ffffffffa00ec9b1 R14: ffff98321b547a58 R15: ffff98321b547a88 [ 1033.208070] FS: 0000000000000000(0000) GS:ffff9832bf380000(0000) knlGS:0000000000000000 [ 1033.208071] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.208072] CR2: 00007ffff7fd6000 CR3: 0000003f67e7a000 CR4: 00000000003607e0 [ 1033.208072] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.208073] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.208074] Call Trace: [ 1033.208076] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.208078] [] _raw_spin_lock+0x30/0x40 [ 1033.208084] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.208092] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.208094] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.208102] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.208126] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.208150] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1033.208152] [] ? check_preempt_curr+0x80/0xa0 [ 1033.208175] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1033.208197] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1033.208228] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1033.208256] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.208259] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.208285] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.208310] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.208311] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.208313] [] ? sched_feat_set+0xf0/0xf0 [ 1033.208337] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.208361] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.208363] [] kthread+0xd1/0xe0 [ 1033.208365] [] ? insert_kthread_work+0x40/0x40 [ 1033.208367] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.208368] [] ? insert_kthread_work+0x40/0x40 [ 1033.208369] Code: [ 1033.208369] 13 [ 1033.208370] 48 [ 1033.208370] c1 [ 1033.208370] ea [ 1033.208371] 0d [ 1033.208371] 48 [ 1033.208372] 98 [ 1033.208372] 83 [ 1033.208372] e2 [ 1033.208373] 30 [ 1033.208373] 48 [ 1033.208373] 81 [ 1033.208374] c2 [ 1033.208374] c0 [ 1033.208375] b8 [ 1033.208375] 01 [ 1033.208375] 00 [ 1033.208376] 48 [ 1033.208376] 03 [ 1033.208377] 14 [ 1033.208377] c5 [ 1033.208377] e0 [ 1033.208378] 17 [ 1033.208378] d5 [ 1033.208378] a0 [ 1033.208379] 4c [ 1033.208379] 89 [ 1033.208379] 02 [ 1033.208380] 41 [ 1033.208380] 8b [ 1033.208381] 40 [ 1033.208381] 08 [ 1033.208381] 85 [ 1033.208382] c0 [ 1033.208382] 75 [ 1033.208383] 0f [ 1033.208383] 0f [ 1033.208383] 1f [ 1033.208384] 44 [ 1033.208384] 00 [ 1033.208385] 00 [ 1033.208385] f3 [ 1033.208385] 90 [ 1033.208386] <41> [ 1033.208386] 8b [ 1033.208387] 40 [ 1033.208387] 08 [ 1033.208387] 85 [ 1033.208388] c0 [ 1033.208388] 74 [ 1033.208388] f6 [ 1033.208389] 4d [ 1033.208389] 8b [ 1033.208389] 08 [ 1033.208390] 4d [ 1033.208390] 85 [ 1033.208390] c9 [ 1033.208391] 74 [ 1033.208391] 04 [ 1033.208391] 41 [ 1033.208392] 0f [ 1033.208392] 18 [ 1033.208393] 09 [ 1033.208393] 8b [ 1033.208393] [ 1033.211999] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [mdt01_007:17310] [ 1033.212000] Modules linked in: [ 1033.212000] osp(OE) [ 1033.212001] mdd(OE) [ 1033.212001] lod(OE) [ 1033.212002] mdt(OE) [ 1033.212002] lfsck(OE) [ 1033.212003] mgs(OE) [ 1033.212003] mgc(OE) [ 1033.212003] osd_zfs(OE) [ 1033.212004] lquota(OE) [ 1033.212004] fid(OE) [ 1033.212004] fld(OE) [ 1033.212004] ptlrpc(OE) [ 1033.212005] obdclass(OE) [ 1033.212005] ko2iblnd(OE) [ 1033.212005] lnet(OE) [ 1033.212006] libcfs(OE) [ 1033.212006] iTCO_wdt [ 1033.212006] iTCO_vendor_support [ 1033.212007] sb_edac [ 1033.212007] intel_powerclamp [ 1033.212007] coretemp [ 1033.212007] intel_rapl [ 1033.212008] iosf_mbi [ 1033.212008] rpcrdma [ 1033.212008] ib_iser [ 1033.212008] kvm [ 1033.212009] irqbypass [ 1033.212009] joydev [ 1033.212009] pcspkr [ 1033.212010] lpc_ich [ 1033.212010] i2c_i801 [ 1033.212010] ioatdma [ 1033.212010] ses [ 1033.212011] enclosure [ 1033.212011] sg [ 1033.212011] ipmi_si [ 1033.212012] ipmi_devintf [ 1033.212012] ipmi_msghandler [ 1033.212012] acpi_power_meter [ 1033.212013] dm_round_robin [ 1033.212013] acpi_cpufreq [ 1033.212013] ib_ipoib [ 1033.212014] rdma_ucm [ 1033.212014] ib_umad [ 1033.212014] sch_fq_codel [ 1033.212015] iw_cxgb4 [ 1033.212015] rdma_cm [ 1033.212015] iw_cm [ 1033.212015] ib_cm [ 1033.212016] iw_cxgb3 [ 1033.212016] zfs(POE) [ 1033.212017] binfmt_misc [ 1033.212017] zunicode(POE) [ 1033.212017] zavl(POE) [ 1033.212018] icp(POE) [ 1033.212018] zcommon(POE) [ 1033.212018] znvpair(POE) [ 1033.212019] spl(OE) [ 1033.212019] msr_safe(OE) [ 1033.212019] ip_tables [ 1033.212020] nfsv3 [ 1033.212020] nfs_acl [ 1033.212020] rpcsec_gss_krb5 [ 1033.212021] auth_rpcgss [ 1033.212021] nfsv4 [ 1033.212021] dns_resolver [ 1033.212022] nfs [ 1033.212022] lockd [ 1033.212022] grace [ 1033.212023] fscache [ 1033.212023] overlay(T) [ 1033.212023] ext4 [ 1033.212024] mbcache [ 1033.212024] jbd2 [ 1033.212024] dm_service_time [ 1033.212024] sd_mod [ 1033.212025] crc_t10dif [ 1033.212025] crct10dif_generic [ 1033.212025] mlx5_ib [ 1033.212026] ib_uverbs [ 1033.212026] ib_core [ 1033.212026] be2iscsi [ 1033.212027] bnx2i [ 1033.212027] cnic [ 1033.212027] uio [ 1033.212027] cxgb4i [ 1033.212028] cxgb4 [ 1033.212028] cxgb3i [ 1033.212028] cxgb3 [ 1033.212028] mdio [ 1033.212029] libcxgbi [ 1033.212029] libcxgb [ 1033.212029] qla4xxx [ 1033.212030] iscsi_boot_sysfs [ 1033.212030] mgag200 [ 1033.212030] i2c_algo_bit [ 1033.212030] 8021q [ 1033.212031] drm_kms_helper [ 1033.212031] garp [ 1033.212031] syscopyarea [ 1033.212032] mrp [ 1033.212032] crct10dif_pclmul [ 1033.212032] stp [ 1033.212032] sysfillrect [ 1033.212033] crct10dif_common [ 1033.212033] sysimgblt [ 1033.212033] crc32_pclmul [ 1033.212034] llc [ 1033.212034] fb_sys_fops [ 1033.212034] crc32c_intel [ 1033.212035] ttm [ 1033.212035] ghash_clmulni_intel [ 1033.212035] mlx5_core [ 1033.212036] drm [ 1033.212036] mxm_wmi [ 1033.212036] ahci [ 1033.212037] ixgbe(OE) [ 1033.212037] aesni_intel [ 1033.212037] mpt3sas [ 1033.212038] mlxfw [ 1033.212038] dm_multipath [ 1033.212038] lrw [ 1033.212039] devlink [ 1033.212039] libahci [ 1033.212039] gf128mul [ 1033.212039] dca [ 1033.212040] glue_helper [ 1033.212040] ablk_helper [ 1033.212040] raid_class [ 1033.212041] ptp [ 1033.212041] drm_panel_orientation_quirks [ 1033.212041] libata [ 1033.212042] cryptd [ 1033.212042] scsi_transport_sas [ 1033.212042] pps_core [ 1033.212043] wmi [ 1033.212043] sunrpc [ 1033.212044] dm_mirror [ 1033.212044] dm_region_hash [ 1033.212044] dm_log [ 1033.212044] dm_mod [ 1033.212045] iscsi_tcp [ 1033.212045] libiscsi_tcp [ 1033.212045] libiscsi [ 1033.212046] scsi_transport_iscsi [ 1033.212046] fuse [ 1033.212046] [ 1033.212048] CPU: 15 PID: 17310 Comm: mdt01_007 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.212049] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.212049] task: ffff98321c6be300 ti: ffff983217874000 task.ti: ffff983217874000 [ 1033.212050] RIP: 0010:[] [ 1033.212052] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1033.212053] RSP: 0018:ffff983217877ab0 EFLAGS: 00000246 [ 1033.212053] RAX: 0000000000000000 RBX: ffff9832bf3c0000 RCX: 0000000000790000 [ 1033.212054] RDX: ffff97f2bfc5b8c0 RSI: 0000000000890000 RDI: ffff97f2a83bcf40 [ 1033.212055] RBP: ffff983217877ab0 R08: ffff9832bf3db8c0 R09: 0000000000000000 [ 1033.212055] R10: 0000000000000002 R11: 0000000000000400 R12: ffff983217877a78 [ 1033.212056] R13: ffffffffa00ec9b1 R14: ffff983217877a58 R15: ffff983217877a88 [ 1033.212057] FS: 0000000000000000(0000) GS:ffff9832bf3c0000(0000) knlGS:0000000000000000 [ 1033.212058] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.212058] CR2: 00007fe543a8a000 CR3: 0000003f74de6000 CR4: 00000000003607e0 [ 1033.212059] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.212060] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.212060] Call Trace: [ 1033.212062] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.212064] [] _raw_spin_lock+0x30/0x40 [ 1033.212069] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.212075] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.212077] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.212083] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.212101] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.212119] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1033.212121] [] ? check_preempt_curr+0x80/0xa0 [ 1033.212138] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1033.212155] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1033.212178] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1033.212205] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.212206] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.212232] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.212257] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.212259] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.212260] [] ? sched_feat_set+0xf0/0xf0 [ 1033.212285] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.212311] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.212312] [] kthread+0xd1/0xe0 [ 1033.212314] [] ? insert_kthread_work+0x40/0x40 [ 1033.212315] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.212316] [] ? insert_kthread_work+0x40/0x40 [ 1033.212317] Code: [ 1033.212317] 13 [ 1033.212318] 48 [ 1033.212318] c1 [ 1033.212318] ea [ 1033.212318] 0d [ 1033.212319] 48 [ 1033.212319] 98 [ 1033.212319] 83 [ 1033.212319] e2 [ 1033.212320] 30 [ 1033.212320] 48 [ 1033.212320] 81 [ 1033.212320] c2 [ 1033.212321] c0 [ 1033.212321] b8 [ 1033.212321] 01 [ 1033.212322] 00 [ 1033.212322] 48 [ 1033.212322] 03 [ 1033.212322] 14 [ 1033.212323] c5 [ 1033.212323] e0 [ 1033.212323] 17 [ 1033.212324] d5 [ 1033.212324] a0 [ 1033.212324] 4c [ 1033.212324] 89 [ 1033.212325] 02 [ 1033.212325] 41 [ 1033.212325] 8b [ 1033.212326] 40 [ 1033.212326] 08 [ 1033.212326] 85 [ 1033.212326] c0 [ 1033.212327] 75 [ 1033.212327] 0f [ 1033.212327] 0f [ 1033.212327] 1f [ 1033.212328] 44 [ 1033.212328] 00 [ 1033.212328] 00 [ 1033.212328] f3 [ 1033.212329] 90 [ 1033.212329] <41> [ 1033.212329] 8b [ 1033.212330] 40 [ 1033.212330] 08 [ 1033.212330] 85 [ 1033.212330] c0 [ 1033.212331] 74 [ 1033.212331] f6 [ 1033.212331] 4d [ 1033.212332] 8b [ 1033.212332] 08 [ 1033.212332] 4d [ 1033.212332] 85 [ 1033.212333] c9 [ 1033.212333] 74 [ 1033.212333] 04 [ 1033.212334] 41 [ 1033.212334] 0f [ 1033.212334] 18 [ 1033.212334] 09 [ 1033.212335] 8b [ 1033.212335] [ 1033.226000] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ll_mgs_0013:10822] [ 1033.226001] Modules linked in: [ 1033.226001] osp(OE) [ 1033.226002] mdd(OE) [ 1033.226002] lod(OE) [ 1033.226003] mdt(OE) [ 1033.226003] lfsck(OE) [ 1033.226004] mgs(OE) [ 1033.226004] mgc(OE) [ 1033.226005] osd_zfs(OE) [ 1033.226005] lquota(OE) [ 1033.226005] fid(OE) [ 1033.226005] fld(OE) [ 1033.226006] ptlrpc(OE) [ 1033.226006] obdclass(OE) [ 1033.226006] ko2iblnd(OE) [ 1033.226007] lnet(OE) [ 1033.226007] libcfs(OE) [ 1033.226007] iTCO_wdt [ 1033.226008] iTCO_vendor_support [ 1033.226008] sb_edac [ 1033.226008] intel_powerclamp [ 1033.226008] coretemp [ 1033.226009] intel_rapl [ 1033.226009] iosf_mbi [ 1033.226009] rpcrdma [ 1033.226009] ib_iser [ 1033.226010] kvm [ 1033.226010] irqbypass [ 1033.226010] joydev [ 1033.226011] pcspkr [ 1033.226011] lpc_ich [ 1033.226011] i2c_i801 [ 1033.226011] ioatdma [ 1033.226012] ses [ 1033.226012] enclosure [ 1033.226012] sg [ 1033.226012] ipmi_si [ 1033.226013] ipmi_devintf [ 1033.226013] ipmi_msghandler [ 1033.226013] acpi_power_meter [ 1033.226014] dm_round_robin [ 1033.226014] acpi_cpufreq [ 1033.226014] ib_ipoib [ 1033.226015] rdma_ucm [ 1033.226015] ib_umad [ 1033.226015] sch_fq_codel [ 1033.226016] iw_cxgb4 [ 1033.226016] rdma_cm [ 1033.226016] iw_cm [ 1033.226016] ib_cm [ 1033.226017] iw_cxgb3 [ 1033.226017] zfs(POE) [ 1033.226017] binfmt_misc [ 1033.226018] zunicode(POE) [ 1033.226018] zavl(POE) [ 1033.226018] icp(POE) [ 1033.226019] zcommon(POE) [ 1033.226019] znvpair(POE) [ 1033.226019] spl(OE) [ 1033.226020] msr_safe(OE) [ 1033.226020] ip_tables [ 1033.226020] nfsv3 [ 1033.226021] nfs_acl [ 1033.226021] rpcsec_gss_krb5 [ 1033.226021] auth_rpcgss [ 1033.226022] nfsv4 [ 1033.226022] dns_resolver [ 1033.226022] nfs [ 1033.226022] lockd [ 1033.226023] grace [ 1033.226023] fscache [ 1033.226023] overlay(T) [ 1033.226024] ext4 [ 1033.226024] mbcache [ 1033.226024] jbd2 [ 1033.226025] dm_service_time [ 1033.226025] sd_mod [ 1033.226025] crc_t10dif [ 1033.226026] crct10dif_generic [ 1033.226026] mlx5_ib [ 1033.226026] ib_uverbs [ 1033.226027] ib_core [ 1033.226027] be2iscsi [ 1033.226027] bnx2i [ 1033.226027] cnic [ 1033.226028] uio [ 1033.226028] cxgb4i [ 1033.226028] cxgb4 [ 1033.226028] cxgb3i [ 1033.226029] cxgb3 [ 1033.226029] mdio [ 1033.226029] libcxgbi [ 1033.226030] libcxgb [ 1033.226030] qla4xxx [ 1033.226030] iscsi_boot_sysfs [ 1033.226030] mgag200 [ 1033.226031] i2c_algo_bit [ 1033.226031] 8021q [ 1033.226031] drm_kms_helper [ 1033.226032] garp [ 1033.226032] syscopyarea [ 1033.226032] mrp [ 1033.226032] crct10dif_pclmul [ 1033.226033] stp [ 1033.226033] sysfillrect [ 1033.226033] crct10dif_common [ 1033.226033] sysimgblt [ 1033.226034] crc32_pclmul [ 1033.226034] llc [ 1033.226034] fb_sys_fops [ 1033.226034] crc32c_intel [ 1033.226035] ttm [ 1033.226035] ghash_clmulni_intel [ 1033.226035] mlx5_core [ 1033.226036] drm [ 1033.226036] mxm_wmi [ 1033.226036] ahci [ 1033.226037] ixgbe(OE) [ 1033.226037] aesni_intel [ 1033.226037] mpt3sas [ 1033.226037] mlxfw [ 1033.226038] dm_multipath [ 1033.226038] lrw [ 1033.226038] devlink [ 1033.226038] libahci [ 1033.226039] gf128mul [ 1033.226039] dca [ 1033.226039] glue_helper [ 1033.226040] ablk_helper [ 1033.226040] raid_class [ 1033.226040] ptp [ 1033.226041] drm_panel_orientation_quirks [ 1033.226041] libata [ 1033.226041] cryptd [ 1033.226041] scsi_transport_sas [ 1033.226042] pps_core [ 1033.226042] wmi [ 1033.226042] sunrpc [ 1033.226043] dm_mirror [ 1033.226043] dm_region_hash [ 1033.226043] dm_log [ 1033.226043] dm_mod [ 1033.226044] iscsi_tcp [ 1033.226044] libiscsi_tcp [ 1033.226044] libiscsi [ 1033.226045] scsi_transport_iscsi [ 1033.226045] fuse [ 1033.226045] [ 1033.226047] CPU: 20 PID: 10822 Comm: ll_mgs_0013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.226047] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.226048] task: ffff98322385a100 ti: ffff983223868000 task.ti: ffff983223868000 [ 1033.226049] RIP: 0010:[] [ 1033.226051] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1033.226051] RSP: 0018:ffff98322386ba50 EFLAGS: 00000246 [ 1033.226052] RAX: 0000000000000000 RBX: ffff97f2bfd00000 RCX: 0000000000a10000 [ 1033.226053] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 1033.226054] RBP: ffff98322386ba50 R08: ffff97f2bfd1b8c0 R09: 0000000000000000 [ 1033.226054] R10: 0000000000000001 R11: 00000000000024c4 R12: ffff98322386ba18 [ 1033.226055] R13: ffffffffa00ec9b1 R14: ffff98322386b9f8 R15: ffff98322386ba28 [ 1033.226056] FS: 0000000000000000(0000) GS:ffff97f2bfd00000(0000) knlGS:0000000000000000 [ 1033.226057] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.226057] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1033.226058] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.226058] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.226059] Call Trace: [ 1033.226061] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.226063] [] _raw_spin_lock+0x30/0x40 [ 1033.226068] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.226075] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.226077] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.226082] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.226103] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.226120] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1033.226121] [] ? enqueue_task_fair+0x208/0x6c0 [ 1033.226123] [] ? check_preempt_curr+0x80/0xa0 [ 1033.226124] [] ? ttwu_do_wakeup+0x19/0x100 [ 1033.226147] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1033.226169] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.226170] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.226190] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.226209] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.226210] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.226211] [] ? sched_feat_set+0xf0/0xf0 [ 1033.226229] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.226231] [] ? __switch_to+0xce/0x5a0 [ 1033.226249] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.226250] [] kthread+0xd1/0xe0 [ 1033.226252] [] ? insert_kthread_work+0x40/0x40 [ 1033.226253] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.226255] [] ? insert_kthread_work+0x40/0x40 [ 1033.226255] Code: [ 1033.226256] 0d [ 1033.226256] 48 [ 1033.226256] 98 [ 1033.226256] 83 [ 1033.226257] e2 [ 1033.226257] 30 [ 1033.226257] 48 [ 1033.226257] 81 [ 1033.226258] c2 [ 1033.226258] c0 [ 1033.226258] b8 [ 1033.226258] 01 [ 1033.226259] 00 [ 1033.226259] 48 [ 1033.226259] 03 [ 1033.226259] 14 [ 1033.226260] c5 [ 1033.226260] e0 [ 1033.226260] 17 [ 1033.226260] d5 [ 1033.226261] a0 [ 1033.226261] 4c [ 1033.226261] 89 [ 1033.226261] 02 [ 1033.226262] 41 [ 1033.226262] 8b [ 1033.226262] 40 [ 1033.226263] 08 [ 1033.226263] 85 [ 1033.226263] c0 [ 1033.226263] 75 [ 1033.226264] 0f [ 1033.226264] 0f [ 1033.226264] 1f [ 1033.226264] 44 [ 1033.226265] 00 [ 1033.226265] 00 [ 1033.226265] f3 [ 1033.226266] 90 [ 1033.226266] 41 [ 1033.226266] 8b [ 1033.226266] 40 [ 1033.226267] 08 [ 1033.226267] <85> [ 1033.226267] c0 [ 1033.226267] 74 [ 1033.226268] f6 [ 1033.226268] 4d [ 1033.226268] 8b [ 1033.226268] 08 [ 1033.226269] 4d [ 1033.226269] 85 [ 1033.226269] c9 [ 1033.226269] 74 [ 1033.226270] 04 [ 1033.226270] 41 [ 1033.226270] 0f [ 1033.226270] 18 [ 1033.226271] 09 [ 1033.226271] 8b [ 1033.226271] 17 [ 1033.226271] 0f [ 1033.226272] b7 [ 1033.226272] c2 [ 1033.226272] [ 1033.232001] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ll_mgs_0014:10826] [ 1033.232001] Modules linked in: [ 1033.232002] osp(OE) [ 1033.232002] mdd(OE) [ 1033.232003] lod(OE) [ 1033.232003] mdt(OE) [ 1033.232004] lfsck(OE) [ 1033.232004] mgs(OE) [ 1033.232004] mgc(OE) [ 1033.232005] osd_zfs(OE) [ 1033.232005] lquota(OE) [ 1033.232005] fid(OE) [ 1033.232006] fld(OE) [ 1033.232006] ptlrpc(OE) [ 1033.232006] obdclass(OE) [ 1033.232007] ko2iblnd(OE) [ 1033.232007] lnet(OE) [ 1033.232007] libcfs(OE) [ 1033.232007] iTCO_wdt [ 1033.232008] iTCO_vendor_support [ 1033.232008] sb_edac [ 1033.232008] intel_powerclamp [ 1033.232009] coretemp [ 1033.232009] intel_rapl [ 1033.232009] iosf_mbi [ 1033.232009] rpcrdma [ 1033.232010] ib_iser [ 1033.232010] kvm [ 1033.232010] irqbypass [ 1033.232010] joydev [ 1033.232011] pcspkr [ 1033.232011] lpc_ich [ 1033.232011] i2c_i801 [ 1033.232012] ioatdma [ 1033.232012] ses [ 1033.232012] enclosure [ 1033.232012] sg [ 1033.232013] ipmi_si [ 1033.232013] ipmi_devintf [ 1033.232013] ipmi_msghandler [ 1033.232014] acpi_power_meter [ 1033.232014] dm_round_robin [ 1033.232014] acpi_cpufreq [ 1033.232014] ib_ipoib [ 1033.232015] rdma_ucm [ 1033.232015] ib_umad [ 1033.232015] sch_fq_codel [ 1033.232016] iw_cxgb4 [ 1033.232016] rdma_cm [ 1033.232016] iw_cm [ 1033.232016] ib_cm [ 1033.232017] iw_cxgb3 [ 1033.232017] zfs(POE) [ 1033.232017] binfmt_misc [ 1033.232018] zunicode(POE) [ 1033.232018] zavl(POE) [ 1033.232018] icp(POE) [ 1033.232019] zcommon(POE) [ 1033.232019] znvpair(POE) [ 1033.232019] spl(OE) [ 1033.232020] msr_safe(OE) [ 1033.232020] ip_tables [ 1033.232020] nfsv3 [ 1033.232021] nfs_acl [ 1033.232021] rpcsec_gss_krb5 [ 1033.232021] auth_rpcgss [ 1033.232022] nfsv4 [ 1033.232022] dns_resolver [ 1033.232022] nfs [ 1033.232022] lockd [ 1033.232023] grace [ 1033.232023] fscache [ 1033.232023] overlay(T) [ 1033.232024] ext4 [ 1033.232024] mbcache [ 1033.232024] jbd2 [ 1033.232024] dm_service_time [ 1033.232025] sd_mod [ 1033.232025] crc_t10dif [ 1033.232025] crct10dif_generic [ 1033.232026] mlx5_ib [ 1033.232026] ib_uverbs [ 1033.232026] ib_core [ 1033.232026] be2iscsi [ 1033.232027] bnx2i [ 1033.232027] cnic [ 1033.232027] uio [ 1033.232027] cxgb4i [ 1033.232028] cxgb4 [ 1033.232028] cxgb3i [ 1033.232028] cxgb3 [ 1033.232028] mdio [ 1033.232029] libcxgbi [ 1033.232029] libcxgb [ 1033.232029] qla4xxx [ 1033.232030] iscsi_boot_sysfs [ 1033.232030] mgag200 [ 1033.232030] i2c_algo_bit [ 1033.232030] 8021q [ 1033.232031] drm_kms_helper [ 1033.232031] garp [ 1033.232031] syscopyarea [ 1033.232031] mrp [ 1033.232032] crct10dif_pclmul [ 1033.232032] stp [ 1033.232032] sysfillrect [ 1033.232033] crct10dif_common [ 1033.232033] sysimgblt [ 1033.232033] crc32_pclmul [ 1033.232034] llc [ 1033.232034] fb_sys_fops [ 1033.232034] crc32c_intel [ 1033.232034] ttm [ 1033.232035] ghash_clmulni_intel [ 1033.232035] mlx5_core [ 1033.232035] drm [ 1033.232036] mxm_wmi [ 1033.232036] ahci [ 1033.232036] ixgbe(OE) [ 1033.232037] aesni_intel [ 1033.232037] mpt3sas [ 1033.232037] mlxfw [ 1033.232038] dm_multipath [ 1033.232038] lrw [ 1033.232038] devlink [ 1033.232038] libahci [ 1033.232039] gf128mul [ 1033.232039] dca [ 1033.232039] glue_helper [ 1033.232039] ablk_helper [ 1033.232040] raid_class [ 1033.232040] ptp [ 1033.232040] drm_panel_orientation_quirks [ 1033.232041] libata [ 1033.232041] cryptd [ 1033.232041] scsi_transport_sas [ 1033.232042] pps_core [ 1033.232042] wmi [ 1033.232042] sunrpc [ 1033.232042] dm_mirror [ 1033.232043] dm_region_hash [ 1033.232043] dm_log [ 1033.232043] dm_mod [ 1033.232043] iscsi_tcp [ 1033.232044] libiscsi_tcp [ 1033.232044] libiscsi [ 1033.232044] scsi_transport_iscsi [ 1033.232044] fuse [ 1033.232045] [ 1033.232046] CPU: 23 PID: 10826 Comm: ll_mgs_0014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.232047] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.232047] task: ffff98322385e300 ti: ffff983225584000 task.ti: ffff983225584000 [ 1033.232048] RIP: 0010:[] [ 1033.232050] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 1033.232051] RSP: 0018:ffff983225587ab0 EFLAGS: 00000246 [ 1033.232052] RAX: 0000000000000000 RBX: ffff97f2bfdc0000 RCX: 0000000000b90000 [ 1033.232052] RDX: ffff97f2bfb5b8c0 RSI: 0000000000290000 RDI: ffff97f2a83bcf40 [ 1033.232053] RBP: ffff983225587ab0 R08: ffff97f2bfddb8c0 R09: 0000000000000000 [ 1033.232053] R10: 0000000000000001 R11: 0000000000002431 R12: ffff983225587a78 [ 1033.232054] R13: ffffffffa00ec9b1 R14: ffff983225587a58 R15: ffff983225587a88 [ 1033.232055] FS: 0000000000000000(0000) GS:ffff97f2bfdc0000(0000) knlGS:0000000000000000 [ 1033.232055] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.232056] CR2: 00007ffff579f624 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1033.232057] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.232057] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.232058] Call Trace: [ 1033.232060] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.232061] [] _raw_spin_lock+0x30/0x40 [ 1033.232067] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.232073] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.232075] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.232080] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.232099] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.232117] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1033.232119] [] ? check_preempt_curr+0x80/0xa0 [ 1033.232136] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1033.232152] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1033.232175] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1033.232196] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.232198] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.232217] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.232235] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.232237] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.232238] [] ? sched_feat_set+0xf0/0xf0 [ 1033.232256] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.232257] [] ? __switch_to+0xce/0x5a0 [ 1033.232275] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.232277] [] kthread+0xd1/0xe0 [ 1033.232278] [] ? insert_kthread_work+0x40/0x40 [ 1033.232280] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.232281] [] ? insert_kthread_work+0x40/0x40 [ 1033.232282] Code: [ 1033.232282] c1 [ 1033.232282] e8 [ 1033.232283] 13 [ 1033.232283] 48 [ 1033.232283] c1 [ 1033.232283] ea [ 1033.232284] 0d [ 1033.232284] 48 [ 1033.232284] 98 [ 1033.232284] 83 [ 1033.232285] e2 [ 1033.232285] 30 [ 1033.232285] 48 [ 1033.232285] 81 [ 1033.232286] c2 [ 1033.232286] c0 [ 1033.232286] b8 [ 1033.232286] 01 [ 1033.232287] 00 [ 1033.232287] 48 [ 1033.232287] 03 [ 1033.232287] 14 [ 1033.232288] c5 [ 1033.232288] e0 [ 1033.232288] 17 [ 1033.232288] d5 [ 1033.232288] a0 [ 1033.232289] 4c [ 1033.232289] 89 [ 1033.232289] 02 [ 1033.232289] 41 [ 1033.232290] 8b [ 1033.232290] 40 [ 1033.232290] 08 [ 1033.232290] 85 [ 1033.232291] c0 [ 1033.232291] 75 [ 1033.232291] 0f [ 1033.232291] 0f [ 1033.232292] 1f [ 1033.232292] 44 [ 1033.232292] 00 [ 1033.232293] 00 [ 1033.232293] [ 1033.232293] 90 [ 1033.232293] 41 [ 1033.232294] 8b [ 1033.232294] 40 [ 1033.232294] 08 [ 1033.232294] 85 [ 1033.232295] c0 [ 1033.232295] 74 [ 1033.232295] f6 [ 1033.232295] 4d [ 1033.232296] 8b [ 1033.232296] 08 [ 1033.232296] 4d [ 1033.232296] 85 [ 1033.232297] c9 [ 1033.232297] 74 [ 1033.232297] 04 [ 1033.232297] 41 [ 1033.232298] 0f [ 1033.232298] 18 [ 1033.232298] [ 1033.244002] NMI watchdog: BUG: soft lockup - CPU#27 stuck for 23s! [mdt01_014:17318] [ 1033.244003] Modules linked in: [ 1033.244003] osp(OE) [ 1033.244004] mdd(OE) [ 1033.244004] lod(OE) [ 1033.244005] mdt(OE) [ 1033.244005] lfsck(OE) [ 1033.244005] mgs(OE) [ 1033.244006] mgc(OE) [ 1033.244006] osd_zfs(OE) [ 1033.244006] lquota(OE) [ 1033.244007] fid(OE) [ 1033.244007] fld(OE) [ 1033.244007] ptlrpc(OE) [ 1033.244008] obdclass(OE) [ 1033.244008] ko2iblnd(OE) [ 1033.244008] lnet(OE) [ 1033.244008] libcfs(OE) [ 1033.244009] iTCO_wdt [ 1033.244009] iTCO_vendor_support [ 1033.244010] sb_edac [ 1033.244010] intel_powerclamp [ 1033.244010] coretemp [ 1033.244010] intel_rapl [ 1033.244011] iosf_mbi [ 1033.244011] rpcrdma [ 1033.244011] ib_iser [ 1033.244012] kvm [ 1033.244012] irqbypass [ 1033.244012] joydev [ 1033.244013] pcspkr [ 1033.244013] lpc_ich [ 1033.244013] i2c_i801 [ 1033.244014] ioatdma [ 1033.244014] ses [ 1033.244014] enclosure [ 1033.244014] sg [ 1033.244015] ipmi_si [ 1033.244015] ipmi_devintf [ 1033.244015] ipmi_msghandler [ 1033.244016] acpi_power_meter [ 1033.244016] dm_round_robin [ 1033.244016] acpi_cpufreq [ 1033.244017] ib_ipoib [ 1033.244017] rdma_ucm [ 1033.244017] ib_umad [ 1033.244017] sch_fq_codel [ 1033.244018] iw_cxgb4 [ 1033.244018] rdma_cm [ 1033.244018] iw_cm [ 1033.244018] ib_cm [ 1033.244019] iw_cxgb3 [ 1033.244019] zfs(POE) [ 1033.244019] binfmt_misc [ 1033.244020] zunicode(POE) [ 1033.244020] zavl(POE) [ 1033.244020] icp(POE) [ 1033.244021] zcommon(POE) [ 1033.244021] znvpair(POE) [ 1033.244021] spl(OE) [ 1033.244022] msr_safe(OE) [ 1033.244022] ip_tables [ 1033.244022] nfsv3 [ 1033.244022] nfs_acl [ 1033.244023] rpcsec_gss_krb5 [ 1033.244023] auth_rpcgss [ 1033.244023] nfsv4 [ 1033.244024] dns_resolver [ 1033.244024] nfs [ 1033.244024] lockd [ 1033.244024] grace [ 1033.244025] fscache [ 1033.244025] overlay(T) [ 1033.244025] ext4 [ 1033.244025] mbcache [ 1033.244026] jbd2 [ 1033.244026] dm_service_time [ 1033.244026] sd_mod [ 1033.244027] crc_t10dif [ 1033.244027] crct10dif_generic [ 1033.244027] mlx5_ib [ 1033.244028] ib_uverbs [ 1033.244028] ib_core [ 1033.244028] be2iscsi [ 1033.244028] bnx2i [ 1033.244029] cnic [ 1033.244029] uio [ 1033.244029] cxgb4i [ 1033.244030] cxgb4 [ 1033.244030] cxgb3i [ 1033.244030] cxgb3 [ 1033.244030] mdio [ 1033.244031] libcxgbi [ 1033.244031] libcxgb [ 1033.244031] qla4xxx [ 1033.244032] iscsi_boot_sysfs [ 1033.244032] mgag200 [ 1033.244032] i2c_algo_bit [ 1033.244032] 8021q [ 1033.244033] drm_kms_helper [ 1033.244033] garp [ 1033.244033] syscopyarea [ 1033.244034] mrp [ 1033.244034] crct10dif_pclmul [ 1033.244034] stp [ 1033.244034] sysfillrect [ 1033.244035] crct10dif_common [ 1033.244035] sysimgblt [ 1033.244035] crc32_pclmul [ 1033.244036] llc [ 1033.244036] fb_sys_fops [ 1033.244036] crc32c_intel [ 1033.244037] ttm [ 1033.244037] ghash_clmulni_intel [ 1033.244037] mlx5_core [ 1033.244037] drm [ 1033.244038] mxm_wmi [ 1033.244038] ahci [ 1033.244038] ixgbe(OE) [ 1033.244039] aesni_intel [ 1033.244039] mpt3sas [ 1033.244039] mlxfw [ 1033.244040] dm_multipath [ 1033.244040] lrw [ 1033.244040] devlink [ 1033.244041] libahci [ 1033.244041] gf128mul [ 1033.244041] dca [ 1033.244041] glue_helper [ 1033.244042] ablk_helper [ 1033.244042] raid_class [ 1033.244042] ptp [ 1033.244043] drm_panel_orientation_quirks [ 1033.244043] libata [ 1033.244043] cryptd [ 1033.244044] scsi_transport_sas [ 1033.244044] pps_core [ 1033.244044] wmi [ 1033.244045] sunrpc [ 1033.244045] dm_mirror [ 1033.244045] dm_region_hash [ 1033.244046] dm_log [ 1033.244046] dm_mod [ 1033.244046] iscsi_tcp [ 1033.244046] libiscsi_tcp [ 1033.244047] libiscsi [ 1033.244047] scsi_transport_iscsi [ 1033.244048] fuse [ 1033.244048] [ 1033.244049] CPU: 27 PID: 17318 Comm: mdt01_014 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.244050] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.244051] task: ffff983228595280 ti: ffff98321be30000 task.ti: ffff98321be30000 [ 1033.244051] RIP: 0010:[] [ 1033.244053] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1033.244054] RSP: 0018:ffff98321be33ab0 EFLAGS: 00000246 [ 1033.244055] RAX: 0000000000000000 RBX: ffff9832bf4c0000 RCX: 0000000000d90000 [ 1033.244055] RDX: ffff97f2bfa9b8c0 RSI: 0000000000110000 RDI: ffff97f2a83bcf40 [ 1033.244056] RBP: ffff98321be33ab0 R08: ffff9832bf4db8c0 R09: 0000000000000000 [ 1033.244057] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98321be33a78 [ 1033.244057] R13: ffff9832bf41ad40 R14: ffffffffa00f0602 R15: ffff98321be33a88 [ 1033.244058] FS: 0000000000000000(0000) GS:ffff9832bf4c0000(0000) knlGS:0000000000000000 [ 1033.244059] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.244059] CR2: 00007ffff76a0d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1033.244060] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.244061] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.244061] Call Trace: [ 1033.244063] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.244065] [] _raw_spin_lock+0x30/0x40 [ 1033.244070] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.244076] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.244078] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.244084] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.244102] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.244120] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1033.244139] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1033.244156] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1033.244173] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1033.244196] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1033.244217] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.244218] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.244238] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.244256] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.244257] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.244258] [] ? sched_feat_set+0xf0/0xf0 [ 1033.244277] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.244295] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.244296] [] kthread+0xd1/0xe0 [ 1033.244298] [] ? insert_kthread_work+0x40/0x40 [ 1033.244299] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.244301] [] ? insert_kthread_work+0x40/0x40 [ 1033.244301] Code: [ 1033.244301] 13 [ 1033.244302] 48 [ 1033.244302] c1 [ 1033.244302] ea [ 1033.244303] 0d [ 1033.244303] 48 [ 1033.244303] 98 [ 1033.244303] 83 [ 1033.244304] e2 [ 1033.244304] 30 [ 1033.244304] 48 [ 1033.244304] 81 [ 1033.244304] c2 [ 1033.244305] c0 [ 1033.244305] b8 [ 1033.244305] 01 [ 1033.244306] 00 [ 1033.244306] 48 [ 1033.244306] 03 [ 1033.244306] 14 [ 1033.244307] c5 [ 1033.244307] e0 [ 1033.244307] 17 [ 1033.244307] d5 [ 1033.244308] a0 [ 1033.244308] 4c [ 1033.244308] 89 [ 1033.244309] 02 [ 1033.244309] 41 [ 1033.244309] 8b [ 1033.244309] 40 [ 1033.244310] 08 [ 1033.244310] 85 [ 1033.244310] c0 [ 1033.244310] 75 [ 1033.244311] 0f [ 1033.244311] 0f [ 1033.244311] 1f [ 1033.244312] 44 [ 1033.244312] 00 [ 1033.244312] 00 [ 1033.244313] f3 [ 1033.244313] 90 [ 1033.244313] <41> [ 1033.244313] 8b [ 1033.244314] 40 [ 1033.244314] 08 [ 1033.244314] 85 [ 1033.244315] c0 [ 1033.244315] 74 [ 1033.244315] f6 [ 1033.244315] 4d [ 1033.244316] 8b [ 1033.244316] 08 [ 1033.244316] 4d [ 1033.244316] 85 [ 1033.244317] c9 [ 1033.244317] 74 [ 1033.244317] 04 [ 1033.244317] 41 [ 1033.244318] 0f [ 1033.244318] 18 [ 1033.244318] 09 [ 1033.244318] 8b [ 1033.244319] [ 1033.249002] NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [mdt01_017:17555] [ 1033.249003] Modules linked in: [ 1033.249003] osp(OE) [ 1033.249004] mdd(OE) [ 1033.249004] lod(OE) [ 1033.249004] mdt(OE) [ 1033.249005] lfsck(OE) [ 1033.249005] mgs(OE) [ 1033.249006] mgc(OE) [ 1033.249006] osd_zfs(OE) [ 1033.249007] lquota(OE) [ 1033.249007] fid(OE) [ 1033.249008] fld(OE) [ 1033.249008] ptlrpc(OE) [ 1033.249008] obdclass(OE) [ 1033.249009] ko2iblnd(OE) [ 1033.249009] lnet(OE) [ 1033.249010] libcfs(OE) [ 1033.249010] iTCO_wdt [ 1033.249011] iTCO_vendor_support [ 1033.249011] sb_edac [ 1033.249011] intel_powerclamp [ 1033.249012] coretemp [ 1033.249012] intel_rapl [ 1033.249012] iosf_mbi [ 1033.249013] rpcrdma [ 1033.249013] ib_iser [ 1033.249013] kvm [ 1033.249014] irqbypass [ 1033.249014] joydev [ 1033.249015] pcspkr [ 1033.249015] lpc_ich [ 1033.249015] i2c_i801 [ 1033.249016] ioatdma [ 1033.249016] ses [ 1033.249016] enclosure [ 1033.249017] sg [ 1033.249017] ipmi_si [ 1033.249018] ipmi_devintf [ 1033.249018] ipmi_msghandler [ 1033.249018] acpi_power_meter [ 1033.249019] dm_round_robin [ 1033.249019] acpi_cpufreq [ 1033.249020] ib_ipoib [ 1033.249020] rdma_ucm [ 1033.249020] ib_umad [ 1033.249021] sch_fq_codel [ 1033.249021] iw_cxgb4 [ 1033.249021] rdma_cm [ 1033.249022] iw_cm [ 1033.249022] ib_cm [ 1033.249023] iw_cxgb3 [ 1033.249023] zfs(POE) [ 1033.249023] binfmt_misc [ 1033.249024] zunicode(POE) [ 1033.249024] zavl(POE) [ 1033.249025] icp(POE) [ 1033.249025] zcommon(POE) [ 1033.249026] znvpair(POE) [ 1033.249026] spl(OE) [ 1033.249026] msr_safe(OE) [ 1033.249027] ip_tables [ 1033.249027] nfsv3 [ 1033.249028] nfs_acl [ 1033.249028] rpcsec_gss_krb5 [ 1033.249029] auth_rpcgss [ 1033.249029] nfsv4 [ 1033.249029] dns_resolver [ 1033.249030] nfs [ 1033.249030] lockd [ 1033.249031] grace [ 1033.249031] fscache [ 1033.249032] overlay(T) [ 1033.249032] ext4 [ 1033.249033] mbcache [ 1033.249033] jbd2 [ 1033.249033] dm_service_time [ 1033.249034] sd_mod [ 1033.249034] crc_t10dif [ 1033.249035] crct10dif_generic [ 1033.249035] mlx5_ib [ 1033.249035] ib_uverbs [ 1033.249036] ib_core [ 1033.249036] be2iscsi [ 1033.249037] bnx2i [ 1033.249037] cnic [ 1033.249037] uio [ 1033.249038] cxgb4i [ 1033.249038] cxgb4 [ 1033.249039] cxgb3i [ 1033.249039] cxgb3 [ 1033.249040] mdio [ 1033.249040] libcxgbi [ 1033.249040] libcxgb [ 1033.249041] qla4xxx [ 1033.249041] iscsi_boot_sysfs [ 1033.249042] mgag200 [ 1033.249042] i2c_algo_bit [ 1033.249042] 8021q [ 1033.249043] drm_kms_helper [ 1033.249043] garp [ 1033.249044] syscopyarea [ 1033.249044] mrp [ 1033.249045] crct10dif_pclmul [ 1033.249045] stp [ 1033.249045] sysfillrect [ 1033.249046] crct10dif_common [ 1033.249046] sysimgblt [ 1033.249047] crc32_pclmul [ 1033.249047] llc [ 1033.249047] fb_sys_fops [ 1033.249048] crc32c_intel [ 1033.249048] ttm [ 1033.249049] ghash_clmulni_intel [ 1033.249049] mlx5_core [ 1033.249049] drm [ 1033.249050] mxm_wmi [ 1033.249050] ahci [ 1033.249050] ixgbe(OE) [ 1033.249051] aesni_intel [ 1033.249051] mpt3sas [ 1033.249052] mlxfw [ 1033.249052] dm_multipath [ 1033.249053] lrw [ 1033.249053] devlink [ 1033.249053] libahci [ 1033.249054] gf128mul [ 1033.249054] dca [ 1033.249055] glue_helper [ 1033.249055] ablk_helper [ 1033.249055] raid_class [ 1033.249056] ptp [ 1033.249056] drm_panel_orientation_quirks [ 1033.249057] libata [ 1033.249057] cryptd [ 1033.249058] scsi_transport_sas [ 1033.249058] pps_core [ 1033.249058] wmi [ 1033.249059] sunrpc [ 1033.249059] dm_mirror [ 1033.249060] dm_region_hash [ 1033.249060] dm_log [ 1033.249060] dm_mod [ 1033.249061] iscsi_tcp [ 1033.249061] libiscsi_tcp [ 1033.249062] libiscsi [ 1033.249062] scsi_transport_iscsi [ 1033.249062] fuse [ 1033.249063] [ 1033.249064] CPU: 29 PID: 17555 Comm: mdt01_017 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.249065] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.249066] task: ffff98321c983180 ti: ffff983218f68000 task.ti: ffff983218f68000 [ 1033.249067] RIP: 0010:[] [ 1033.249069] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1033.249070] RSP: 0018:ffff983218f6bab0 EFLAGS: 00000246 [ 1033.249071] RAX: 0000000000000000 RBX: ffff9832bf540000 RCX: 0000000000e90000 [ 1033.249072] RDX: ffff97f2bfc1b8c0 RSI: 0000000000810000 RDI: ffff97f2a83bcf40 [ 1033.249073] RBP: ffff983218f6bab0 R08: ffff9832bf55b8c0 R09: 0000000000000000 [ 1033.249074] R10: 0000000000000002 R11: ffff97f2bef3acb8 R12: ffff983218f6ba78 [ 1033.249075] R13: ffffffffa00ec9b1 R14: ffff983218f6ba58 R15: ffff983218f6ba88 [ 1033.249076] FS: 0000000000000000(0000) GS:ffff9832bf540000(0000) knlGS:0000000000000000 [ 1033.249077] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.249078] CR2: 00007fe543c54000 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1033.249079] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.249079] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.249080] Call Trace: [ 1033.249082] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.249084] [] _raw_spin_lock+0x30/0x40 [ 1033.249090] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.249098] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.249101] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.249108] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.249132] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.249157] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1033.249159] [] ? check_preempt_curr+0x80/0xa0 [ 1033.249181] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1033.249204] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1033.249235] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1033.249263] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.249265] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.249291] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.249316] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.249318] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.249380] iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1033.249382] CPU: 2 PID: 17594 Comm: mdt00_013 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.249382] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.249383] task: ffff98323057b180 ti: ffff98321b874000 task.ti: ffff98321b874000 [ 1033.249386] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1033.249386] RSP: 0018:ffff98321b877a50 EFLAGS: 00000246 [ 1033.249387] RAX: 0000000000000000 RBX: ffff97f2bfa80000 RCX: 0000000000110000 [ 1033.249387] RDX: ffff97f2bfddb8c0 RSI: 0000000000b90000 RDI: ffff97f2a83bcf40 [ 1033.249388] RBP: ffff98321b877a50 R08: ffff97f2bfa9b8c0 R09: 0000000000000000 [ 1033.249388] R10: 0000000000000001 R11: 000000000000242e R12: ffff98321b877a18 [ 1033.249389] R13: ffff97f2bfc9ad40 R14: ffffffffa00f0602 R15: ffff98321b877a28 [ 1033.249390] FS: 0000000000000000(0000) GS:ffff97f2bfa80000(0000) knlGS:0000000000000000 [ 1033.249391] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.249391] CR2: 00007ffff6d792bc CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1033.249392] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.249392] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.249393] Call Trace: [ 1033.249395] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.249397] [] _raw_spin_lock+0x30/0x40 [ 1033.249402] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.249408] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.249410] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.249416] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.249434] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.249450] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1033.249468] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1033.249469] [] ? enqueue_task_fair+0x208/0x6c0 [ 1033.249471] [] ? sched_clock_cpu+0xa5/0xe0 [ 1033.249472] [] ? check_preempt_curr+0x80/0xa0 [ 1033.249473] [] ? ttwu_do_wakeup+0x19/0x100 [ 1033.249496] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1033.249518] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.249521] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.249541] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.249559] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.249560] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.249561] [] ? sched_feat_set+0xf0/0xf0 [ 1033.249579] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.249581] [] ? __switch_to+0xce/0x5a0 [ 1033.249599] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.249601] [] kthread+0xd1/0xe0 [ 1033.249602] [] ? insert_kthread_work+0x40/0x40 [ 1033.249603] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.249605] [] ? insert_kthread_work+0x40/0x40 [ 1033.249619] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1033.251003] NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [mdt01_018:17592] [ 1033.251032] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1033.251053] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1033.251054] CPU: 30 PID: 17592 Comm: mdt01_018 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1033.251055] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1033.251056] task: ffff983230579080 ti: ffff9832469b0000 task.ti: ffff9832469b0000 [ 1033.251059] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1033.251060] RSP: 0018:ffff9832469b3a50 EFLAGS: 00000246 [ 1033.251061] RAX: 0000000000000000 RBX: ffff9832bf580000 RCX: 0000000000f10000 [ 1033.251062] RDX: ffff97f2bfd1b8c0 RSI: 0000000000a10000 RDI: ffff97f2a83bcf40 [ 1033.251062] RBP: ffff9832469b3a50 R08: ffff9832bf59b8c0 R09: 0000000000000000 [ 1033.251063] R10: 0000000000000002 R11: ffff983211336600 R12: ffff9832469b3a18 [ 1033.251064] R13: ffffffffa00ec9b1 R14: ffff9832469b39f8 R15: ffff9832469b3a28 [ 1033.251065] FS: 0000000000000000(0000) GS:ffff9832bf580000(0000) knlGS:0000000000000000 [ 1033.251066] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1033.251067] CR2: 00007ffff7fc5010 CR3: 0000003f74514000 CR4: 00000000003607e0 [ 1033.251068] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1033.251069] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1033.251069] Call Trace: [ 1033.251071] [] queued_spin_lock_slowpath+0xb/0xf [ 1033.251073] [] _raw_spin_lock+0x30/0x40 [ 1033.251079] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1033.251087] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1033.251089] [] ? wake_up_atomic_t+0x30/0x30 [ 1033.251097] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1033.251121] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1033.251144] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1033.251146] [] ? check_preempt_curr+0x80/0xa0 [ 1033.251148] [] ? ttwu_do_wakeup+0x19/0x100 [ 1033.251178] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1033.251206] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1033.251208] [] ? __getnstimeofday64+0x3f/0xd0 [ 1033.251234] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1033.251258] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1033.251259] [] ? __wake_up_common_lock+0x91/0xc0 [ 1033.251261] [] ? sched_feat_set+0xf0/0xf0 [ 1033.251285] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1033.251309] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1033.251311] [] kthread+0xd1/0xe0 [ 1033.251312] [] ? insert_kthread_work+0x40/0x40 [ 1033.251314] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1033.251316] [] ? insert_kthread_work+0x40/0x40 [ 1033.251335] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1034.252576] Lustre: ls1-MDT0000: Client 845648b3-aac0-f60c-9190-d0921c01a074 (at 172.19.3.152@o2ib600) reconnecting [ 1034.252577] Lustre: Skipped 1592 previous similar messages [ 1037.525654] [] ? sched_feat_set+0xf0/0xf0 [ 1037.532478] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1037.541513] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1037.551782] [] kthread+0xd1/0xe0 [ 1037.557710] [] ? insert_kthread_work+0x40/0x40 [ 1037.564997] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1037.574314] [] ? insert_kthread_work+0x40/0x40 [ 1037.581599] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1038.671770] Lustre: MGS: Export ffff97f26f52c000 already connecting from 192.168.128.19@o2ib42 [ 1045.217382] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [mdt00_031:18467] [ 1045.221383] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [mdt00_030:16183] [ 1045.221422] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1045.221449] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1045.221452] CPU: 17 PID: 16183 Comm: mdt00_030 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1045.221452] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1045.221453] task: ffff97f250d53180 ti: ffff97f250d78000 task.ti: ffff97f250d78000 [ 1045.221459] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x120/0x200 [ 1045.221460] RSP: 0018:ffff97f250d7bab0 EFLAGS: 00000246 [ 1045.221460] RAX: 0000000000000000 RBX: ffff97f2bfc40000 RCX: 0000000000890000 [ 1045.221461] RDX: ffff97f2bfd9b8c0 RSI: 0000000000b10000 RDI: ffff97f2a83bcf40 [ 1045.221461] RBP: ffff97f250d7bab0 R08: ffff97f2bfc5b8c0 R09: 0000000000000000 [ 1045.221462] R10: 0000000000000001 R11: 0000000000000400 R12: ffff97f250d7ba78 [ 1045.221463] R13: ffffffffa00ec9b1 R14: ffff97f250d7ba58 R15: ffff97f250d7ba88 [ 1045.221464] FS: 0000000000000000(0000) GS:ffff97f2bfc40000(0000) knlGS:0000000000000000 [ 1045.221464] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1045.221465] CR2: 00007ffff75f34b1 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1045.221466] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1045.221466] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1045.221467] Call Trace: [ 1045.221472] [] queued_spin_lock_slowpath+0xb/0xf [ 1045.221475] [] _raw_spin_lock+0x30/0x40 [ 1045.221488] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1045.221492] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1045.221506] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1045.221509] [] ? wake_up_atomic_t+0x30/0x30 [ 1045.221514] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1045.221560] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1045.221579] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1045.221581] [] ? check_preempt_curr+0x80/0xa0 [ 1045.221598] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1045.221615] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1045.221646] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1045.221668] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1045.221670] [] ? __getnstimeofday64+0x3f/0xd0 [ 1045.221694] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1045.221719] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1045.221721] [] ? __wake_up_common_lock+0x91/0xc0 [ 1045.221722] [] ? sched_feat_set+0xf0/0xf0 [ 1045.221748] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1045.221750] [] ? __switch_to+0xce/0x5a0 [ 1045.221776] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1045.221778] [] kthread+0xd1/0xe0 [ 1045.221779] [] ? insert_kthread_work+0x40/0x40 [ 1045.221781] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1045.221782] [] ? insert_kthread_work+0x40/0x40 [ 1045.221796] Code: c1 e8 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 [ 1045.735198] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1045.815042] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1045.876126] CPU: 16 PID: 18467 Comm: mdt00_031 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1045.890490] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1045.902817] task: ffff97f249f7a100 ti: ffff97f249078000 task.ti: ffff97f249078000 [ 1045.911652] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1045.923214] RSP: 0018:ffff97f24907bab0 EFLAGS: 00000246 [ 1045.929624] RAX: 0000000000000000 RBX: ffff97f2bfc00000 RCX: 0000000000810000 [ 1045.938073] RDX: ffff9832bf35b8c0 RSI: 0000000000690000 RDI: ffff97f2a83bcf40 [ 1045.946522] RBP: ffff97f24907bab0 R08: ffff97f2bfc1b8c0 R09: 0000000000000000 [ 1045.954969] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f24907ba78 [ 1045.963418] R13: ffffffffa00ec9b1 R14: ffff97f24907ba58 R15: ffff97f24907ba88 [ 1045.971865] FS: 0000000000000000(0000) GS:ffff97f2bfc00000(0000) knlGS:0000000000000000 [ 1045.981379] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1045.988274] CR2: 00007ffff7ad2d70 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1045.996721] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1046.005169] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1046.013616] Call Trace: [ 1046.016828] [] queued_spin_lock_slowpath+0xb/0xf [ 1046.024306] [] _raw_spin_lock+0x30/0x40 [ 1046.030917] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1046.038594] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1046.046556] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1046.055102] [] ? wake_up_atomic_t+0x30/0x30 [ 1046.062102] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1046.069509] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1046.077787] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1046.085776] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1046.094440] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1046.102320] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1046.109635] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1046.117820] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1046.126755] [] ? __getnstimeofday64+0x3f/0xd0 [ 1046.133967] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1046.143021] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1046.151082] [] ? __wake_up_common_lock+0x91/0xc0 [ 1046.158561] [] ? sched_feat_set+0xf0/0xf0 [ 1046.165384] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1046.172863] [] ? __switch_to+0xce/0x5a0 [ 1046.179493] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1046.188232] [] kthread+0xd1/0xe0 [ 1046.194159] [] ? insert_kthread_work+0x40/0x40 [ 1046.201443] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1046.209213] [] ? insert_kthread_work+0x40/0x40 [ 1046.216497] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 1048.206727] INFO: rcu_sched self-detected stall on CPU [ 1048.206730] INFO: rcu_sched self-detected stall on CPU [ 1048.206731] { [ 1048.206731] 13 [ 1048.206732] } [ 1048.206734] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.206735] Task dump for CPU 2: [ 1048.206736] mdt00_013 R [ 1048.206736] running task [ 1048.206737] 0 17594 2 0x00000088 [ 1048.206738] Call Trace: [ 1048.206746] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.206750] [] ? _raw_spin_lock+0x30/0x40 [ 1048.206765] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.206787] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.206791] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.206799] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.206838] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.206861] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.206886] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.206889] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.206891] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.206893] [] ? check_preempt_curr+0x80/0xa0 [ 1048.206895] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.206931] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.206959] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.206962] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.206988] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.207012] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.207014] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.207016] [] ? sched_feat_set+0xf0/0xf0 [ 1048.207040] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.207043] [] ? __switch_to+0xce/0x5a0 [ 1048.207067] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.207069] [] ? kthread+0xd1/0xe0 [ 1048.207071] [] ? insert_kthread_work+0x40/0x40 [ 1048.207074] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.207075] [] ? insert_kthread_work+0x40/0x40 [ 1048.207076] Task dump for CPU 8: [ 1048.207077] mdt01_022 R [ 1048.207077] running task [ 1048.207078] 0 17601 2 0x00000088 [ 1048.207078] Call Trace: [ 1048.207082] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.207084] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.207086] [] ? _raw_spin_lock+0x30/0x40 [ 1048.207092] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.207101] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.207103] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.207110] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.207134] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.207158] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.207183] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.207206] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.207229] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.207260] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.207288] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.207289] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.207316] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.207340] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.207342] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.207343] [] ? sched_feat_set+0xf0/0xf0 [ 1048.207367] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.207391] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.207393] [] ? kthread+0xd1/0xe0 [ 1048.207395] [] ? insert_kthread_work+0x40/0x40 [ 1048.207396] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.207398] [] ? insert_kthread_work+0x40/0x40 [ 1048.207399] Task dump for CPU 13: [ 1048.207399] mdt01_003 R [ 1048.207400] running task [ 1048.207400] 0 10825 2 0x00000088 [ 1048.207401] Call Trace: [ 1048.207403] [ 1048.207405] [] sched_show_task+0xbf/0x120 [ 1048.207407] [] dump_cpu_task+0x39/0x70 [ 1048.207411] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 1048.207414] [] rcu_check_callbacks+0x482/0x770 [ 1048.207418] [] update_process_times+0x46/0x80 [ 1048.207422] [] tick_sched_handle+0x30/0x70 [ 1048.207424] [] tick_sched_timer+0x39/0x80 [ 1048.207426] [] __hrtimer_run_queues+0x13e/0x2f0 [ 1048.207428] [] ? tick_sched_do_timer+0x50/0x50 [ 1048.207429] [] hrtimer_interrupt+0xb9/0x1f0 [ 1048.207432] [] local_apic_timer_interrupt+0x3b/0x60 [ 1048.207435] [] smp_apic_timer_interrupt+0x43/0x60 [ 1048.207437] [] apic_timer_interrupt+0x16a/0x170 [ 1048.207437] [ 1048.207439] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.207441] [] queued_spin_lock_slowpath+0xb/0xf [ 1048.207443] [] _raw_spin_lock+0x30/0x40 [ 1048.207449] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.207457] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.207459] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.207467] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.207490] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.207512] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.207514] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.207516] [] ? check_preempt_curr+0x80/0xa0 [ 1048.207517] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.207547] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.207576] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.207577] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.207603] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.207627] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.207629] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.207630] [] ? sched_feat_set+0xf0/0xf0 [ 1048.207654] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.207678] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.207680] [] kthread+0xd1/0xe0 [ 1048.207682] [] ? insert_kthread_work+0x40/0x40 [ 1048.207683] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1048.207685] [] ? insert_kthread_work+0x40/0x40 [ 1048.207686] Task dump for CPU 14: [ 1048.207686] mdt01_016 R [ 1048.207687] running task [ 1048.207687] 0 17554 2 0x00000088 [ 1048.207688] Call Trace: [ 1048.207690] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.207692] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.207694] [] ? _raw_spin_lock+0x30/0x40 [ 1048.207700] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.207708] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.207710] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.207717] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.207741] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.207742] INFO: rcu_sched self-detected stall on CPU [ 1048.207742] INFO: rcu_sched self-detected stall on CPU [ 1048.207743] INFO: rcu_sched self-detected stall on CPU [ 1048.207744] INFO: rcu_sched self-detected stall on CPU [ 1048.207745] INFO: rcu_sched self-detected stall on CPU [ 1048.207746] INFO: rcu_sched self-detected stall on CPU [ 1048.207747] INFO: rcu_sched self-detected stall on CPU [ 1048.207772] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.207772] { [ 1048.207773] { [ 1048.207773] { [ 1048.207773] { [ 1048.207774] { [ 1048.207775] { [ 1048.207775] { [ 1048.207777] [] ? check_preempt_curr+0x80/0xa0 [ 1048.207778] 8 [ 1048.207778] 14 [ 1048.207779] 15 [ 1048.207780] 30 [ 1048.207781] 2 [ 1048.207781] 23 [ 1048.207782] 29 [ 1048.207805] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.207805] } [ 1048.207806] } [ 1048.207806] } [ 1048.207807] } [ 1048.207808] } [ 1048.207808] } [ 1048.207809] } [ 1048.207832] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.207833] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207834] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207835] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207836] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207837] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207838] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207840] (t=240003 jiffies g=29074 c=29073 q=216843) [ 1048.207870] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.207897] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.207899] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.207926] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.207950] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.207952] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.207953] [] ? sched_feat_set+0xf0/0xf0 [ 1048.207977] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.208001] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.208002] [] ? kthread+0xd1/0xe0 [ 1048.208004] [] ? insert_kthread_work+0x40/0x40 [ 1048.208006] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.208007] [] ? insert_kthread_work+0x40/0x40 [ 1048.208008] Task dump for CPU 15: [ 1048.208009] mdt01_007 R [ 1048.208009] running task [ 1048.208010] 0 17310 2 0x00000088 [ 1048.208010] Call Trace: [ 1048.208012] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.208014] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.208016] [] ? _raw_spin_lock+0x30/0x40 [ 1048.208022] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.208030] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.208032] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.208040] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.208063] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.208086] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.208088] [] ? check_preempt_curr+0x80/0xa0 [ 1048.208111] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.208134] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.208164] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.208191] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.208193] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.208218] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.208242] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.208244] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.208245] [] ? sched_feat_set+0xf0/0xf0 [ 1048.208269] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.208295] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.208297] [] ? kthread+0xd1/0xe0 [ 1048.208299] [] ? insert_kthread_work+0x40/0x40 [ 1048.208300] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.208302] [] ? insert_kthread_work+0x40/0x40 [ 1048.208303] Task dump for CPU 20: [ 1048.208304] ll_mgs_0013 R [ 1048.208305] Task dump for CPU 2: [ 1048.208305] running task [ 1048.208306] mdt00_013 R [ 1048.208307] 0 10822 2 0x00000088 [ 1048.208307] running task [ 1048.208307] Call Trace: [ 1048.208308] 0 17594 2 0x00000088 [ 1048.208310] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.208311] Call Trace: [ 1048.208313] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.208315] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.208317] [] ? _raw_spin_lock+0x30/0x40 [ 1048.208318] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.208324] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 1048.208326] [] ? _raw_spin_lock+0x30/0x40 [ 1048.208331] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1048.208336] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.208344] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1048.208350] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.208352] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.208354] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.208362] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.208367] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.208390] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.208409] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.208431] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.208447] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.208449] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.208466] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.208468] [] ? check_preempt_curr+0x80/0xa0 [ 1048.208470] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.208471] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.208473] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.208502] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.208504] [] ? check_preempt_curr+0x80/0xa0 [ 1048.208531] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.208532] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.208534] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.208557] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.208582] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.208603] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.208627] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.208628] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.208630] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.208649] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.208650] [] ? sched_feat_set+0xf0/0xf0 [ 1048.208669] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.208692] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.208694] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.208696] [] ? __switch_to+0xce/0x5a0 [ 1048.208697] [] ? sched_feat_set+0xf0/0xf0 [ 1048.208720] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.208740] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.208742] [] ? kthread+0xd1/0xe0 [ 1048.208743] [] ? __switch_to+0xce/0x5a0 [ 1048.208745] [] ? insert_kthread_work+0x40/0x40 [ 1048.208762] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.208764] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.208766] [] ? kthread+0xd1/0xe0 [ 1048.208767] [] ? insert_kthread_work+0x40/0x40 [ 1048.208769] [] ? insert_kthread_work+0x40/0x40 [ 1048.208769] Task dump for CPU 23: [ 1048.208770] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.208771] ll_mgs_0014 R [ 1048.208773] [] ? insert_kthread_work+0x40/0x40 [ 1048.208773] running task [ 1048.208773] Task dump for CPU 8: [ 1048.208774] 0 10826 2 0x00000088 [ 1048.208775] mdt01_022 R [ 1048.208775] Call Trace: [ 1048.208776] running task [ 1048.208777] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.208778] 0 17601 2 0x00000088 [ 1048.208780] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.208781] Call Trace: [ 1048.208782] [] ? _raw_spin_lock+0x30/0x40 [ 1048.208783] [ 1048.208789] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.208790] [] sched_show_task+0xbf/0x120 [ 1048.208798] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.208800] [] dump_cpu_task+0x39/0x70 [ 1048.208802] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.208803] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 1048.208811] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.208812] [] rcu_check_callbacks+0x482/0x770 [ 1048.208837] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.208839] [] update_process_times+0x46/0x80 [ 1048.208862] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.208864] [] tick_sched_handle+0x30/0x70 [ 1048.208866] [] ? check_preempt_curr+0x80/0xa0 [ 1048.208867] [] tick_sched_timer+0x39/0x80 [ 1048.208890] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.208891] [] __hrtimer_run_queues+0x13e/0x2f0 [ 1048.208913] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.208915] [] ? tick_sched_do_timer+0x50/0x50 [ 1048.208945] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.208946] [] hrtimer_interrupt+0xb9/0x1f0 [ 1048.208973] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.208974] [] local_apic_timer_interrupt+0x3b/0x60 [ 1048.208976] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.208977] [] smp_apic_timer_interrupt+0x43/0x60 [ 1048.209004] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.209005] [] apic_timer_interrupt+0x16a/0x170 [ 1048.209029] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.209030] [ 1048.209031] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.209033] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 1048.209034] [] ? sched_feat_set+0xf0/0xf0 [ 1048.209036] [] queued_spin_lock_slowpath+0xb/0xf [ 1048.209060] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.209061] [] _raw_spin_lock+0x30/0x40 [ 1048.209063] [] ? __switch_to+0xce/0x5a0 [ 1048.209067] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.209091] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.209097] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.209098] [] ? kthread+0xd1/0xe0 [ 1048.209100] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.209102] [] ? insert_kthread_work+0x40/0x40 [ 1048.209107] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.209109] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.209126] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.209128] [] ? insert_kthread_work+0x40/0x40 [ 1048.209145] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.209146] Task dump for CPU 29: [ 1048.209164] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.209164] mdt01_017 R [ 1048.209182] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.209182] running task [ 1048.209183] 0 17555 2 0x00000088 [ 1048.209200] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.209200] Call Trace: [ 1048.209223] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.209224] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.209246] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.209248] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.209249] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.209251] [] ? _raw_spin_lock+0x30/0x40 [ 1048.209270] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.209276] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.209294] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.209302] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.209303] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.209305] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.209306] [] ? sched_feat_set+0xf0/0xf0 [ 1048.209313] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.209331] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.209355] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.209372] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.209396] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.209398] [] kthread+0xd1/0xe0 [ 1048.209399] [] ? check_preempt_curr+0x80/0xa0 [ 1048.209401] [] ? insert_kthread_work+0x40/0x40 [ 1048.209423] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.209425] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1048.209447] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.209449] [] ? insert_kthread_work+0x40/0x40 [ 1048.209478] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.209479] Task dump for CPU 13: [ 1048.209506] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.209506] mdt01_003 R [ 1048.209508] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.209508] running task [ 1048.209509] 0 10825 2 0x00000088 [ 1048.209534] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.209535] Call Trace: [ 1048.209559] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.209560] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.209562] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.209564] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.209565] [] ? sched_feat_set+0xf0/0xf0 [ 1048.209566] [] ? _raw_spin_lock+0x30/0x40 [ 1048.209590] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.209595] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.209618] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.209624] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.209626] [] ? kthread+0xd1/0xe0 [ 1048.209628] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.209630] [] ? insert_kthread_work+0x40/0x40 [ 1048.209635] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.209637] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.209654] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.209655] [] ? insert_kthread_work+0x40/0x40 [ 1048.209671] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.209671] Task dump for CPU 30: [ 1048.209672] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.209673] mdt01_018 R [ 1048.209674] [] ? check_preempt_curr+0x80/0xa0 [ 1048.209675] running task [ 1048.209676] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.209677] 0 17592 2 0x00000088 [ 1048.209699] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.209700] Call Trace: [ 1048.209720] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.209722] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.209723] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.209725] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.209745] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.209747] [] ? _raw_spin_lock+0x30/0x40 [ 1048.209765] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.209771] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.209772] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.209780] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.209781] [] ? sched_feat_set+0xf0/0xf0 [ 1048.209783] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.209801] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.209808] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.209826] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.209849] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.209850] [] ? kthread+0xd1/0xe0 [ 1048.209873] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.209874] [] ? insert_kthread_work+0x40/0x40 [ 1048.209876] [] ? check_preempt_curr+0x80/0xa0 [ 1048.209877] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.209879] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.209880] [] ? insert_kthread_work+0x40/0x40 [ 1048.209910] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.209910] Task dump for CPU 14: [ 1048.209938] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.209938] mdt01_016 R [ 1048.209940] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.209940] running task [ 1048.209941] 0 17554 2 0x00000088 [ 1048.209966] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.209967] Call Trace: [ 1048.209991] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.209993] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.209994] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.209996] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.209997] [] ? sched_feat_set+0xf0/0xf0 [ 1048.209999] [] ? _raw_spin_lock+0x30/0x40 [ 1048.210023] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.210027] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.210051] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.210057] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.210059] [] ? kthread+0xd1/0xe0 [ 1048.210060] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.210062] [] ? insert_kthread_work+0x40/0x40 [ 1048.210067] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.210069] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.210086] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.210088] [] ? insert_kthread_work+0x40/0x40 [ 1048.210112] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.210113] [] ? check_preempt_curr+0x80/0xa0 [ 1048.210137] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.210161] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.210188] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.210215] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.210216] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.210241] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.210266] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.210268] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.210269] [] ? sched_feat_set+0xf0/0xf0 [ 1048.210294] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.210320] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.210321] [] ? kthread+0xd1/0xe0 [ 1048.210323] [] ? insert_kthread_work+0x40/0x40 [ 1048.210324] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.210326] [] ? insert_kthread_work+0x40/0x40 [ 1048.210326] Task dump for CPU 15: [ 1048.210327] mdt01_007 R [ 1048.210327] running task [ 1048.210328] 0 17310 2 0x00000088 [ 1048.210328] Call Trace: [ 1048.210329] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.210331] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.210332] [] ? _raw_spin_lock+0x30/0x40 [ 1048.210337] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.210344] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.210346] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.210352] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.210376] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.210400] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.210401] [] ? check_preempt_curr+0x80/0xa0 [ 1048.210425] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.210449] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.210476] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.210503] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.210504] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.210529] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.210554] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.210556] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.210557] [] ? sched_feat_set+0xf0/0xf0 [ 1048.210582] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.210608] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.210609] [] ? kthread+0xd1/0xe0 [ 1048.210611] [] ? insert_kthread_work+0x40/0x40 [ 1048.210612] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.210613] [] ? insert_kthread_work+0x40/0x40 [ 1048.210614] Task dump for CPU 20: [ 1048.210614] Task dump for CPU 2: [ 1048.210615] ll_mgs_0013 R [ 1048.210616] mdt00_013 R [ 1048.210616] running task [ 1048.210616] running task [ 1048.210617] 0 10822 2 0x00000088 [ 1048.210618] 0 17594 2 0x00000088 [ 1048.210618] Call Trace: [ 1048.210619] Call Trace: [ 1048.210620] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.210623] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.210625] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.210627] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.210628] [] ? _raw_spin_lock+0x30/0x40 [ 1048.210629] [] ? _raw_spin_lock+0x30/0x40 [ 1048.210634] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 1048.210641] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.210645] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1048.210653] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.210660] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1048.210662] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.210663] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.210671] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.210677] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.210701] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.210724] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.210747] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.210763] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.210787] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.210788] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.210790] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.210791] [] ? check_preempt_curr+0x80/0xa0 [ 1048.210793] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.210795] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.210796] [] ? check_preempt_curr+0x80/0xa0 [ 1048.210819] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.210821] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.210842] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.210872] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.210874] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.210901] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.210921] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.210923] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.210941] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.210967] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.210968] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.210992] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.210993] [] ? sched_feat_set+0xf0/0xf0 [ 1048.210995] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.211013] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.211014] [] ? sched_feat_set+0xf0/0xf0 [ 1048.211016] [] ? __switch_to+0xce/0x5a0 [ 1048.211040] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.211057] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.211059] [] ? __switch_to+0xce/0x5a0 [ 1048.211060] [] ? kthread+0xd1/0xe0 [ 1048.211084] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.211085] [] ? insert_kthread_work+0x40/0x40 [ 1048.211087] [] ? kthread+0xd1/0xe0 [ 1048.211088] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.211090] [] ? insert_kthread_work+0x40/0x40 [ 1048.211091] [] ? insert_kthread_work+0x40/0x40 [ 1048.211093] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.211093] Task dump for CPU 23: [ 1048.211095] [] ? insert_kthread_work+0x40/0x40 [ 1048.211095] ll_mgs_0014 R [ 1048.211096] Task dump for CPU 8: [ 1048.211096] running task [ 1048.211097] mdt01_022 R [ 1048.211098] 0 10826 2 0x00000088 [ 1048.211098] running task [ 1048.211098] Call Trace: [ 1048.211099] 0 17601 2 0x00000088 [ 1048.211101] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.211101] Call Trace: [ 1048.211103] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.211105] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 1048.211106] [] ? _raw_spin_lock+0x30/0x40 [ 1048.211108] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.211112] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.211114] [] ? _raw_spin_lock+0x30/0x40 [ 1048.211120] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.211126] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.211128] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.211136] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.211141] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.211143] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.211161] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.211168] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.211186] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.211209] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.211210] [] ? check_preempt_curr+0x80/0xa0 [ 1048.211234] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.211251] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.211275] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.211291] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.211315] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.211336] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.211359] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.211380] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.211410] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.211411] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.211438] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.211457] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.211459] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.211482] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.211507] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.211509] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.211533] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.211534] [] ? sched_feat_set+0xf0/0xf0 [ 1048.211536] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.211561] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.211563] [] ? sched_feat_set+0xf0/0xf0 [ 1048.211564] [] ? __switch_to+0xce/0x5a0 [ 1048.211589] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.211614] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.211638] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.211639] [] ? kthread+0xd1/0xe0 [ 1048.211641] [] ? kthread+0xd1/0xe0 [ 1048.211643] [] ? insert_kthread_work+0x40/0x40 [ 1048.211644] [] ? insert_kthread_work+0x40/0x40 [ 1048.211646] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.211647] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.211649] [] ? insert_kthread_work+0x40/0x40 [ 1048.211650] [] ? insert_kthread_work+0x40/0x40 [ 1048.211651] Task dump for CPU 29: [ 1048.211651] Task dump for CPU 13: [ 1048.211652] mdt01_017 R [ 1048.211652] mdt01_003 R [ 1048.211652] running task [ 1048.211653] running task [ 1048.211654] 0 17555 2 0x00000088 [ 1048.211654] 0 10825 2 0x00000088 [ 1048.211655] Call Trace: [ 1048.211655] Call Trace: [ 1048.211657] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.211658] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.211660] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.211662] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.211663] [] ? _raw_spin_lock+0x30/0x40 [ 1048.211665] [] ? _raw_spin_lock+0x30/0x40 [ 1048.211670] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.211676] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.211682] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.211690] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.211692] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.211694] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.211700] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.211708] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.211732] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.211756] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.211774] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.211797] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.211798] [] ? check_preempt_curr+0x80/0xa0 [ 1048.211800] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.211817] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.211819] [] ? check_preempt_curr+0x80/0xa0 [ 1048.211835] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.211837] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.211859] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.211889] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.211910] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.211937] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.211939] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.211940] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.211960] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.211985] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.212003] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.212027] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.212028] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.212030] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.212031] [] ? sched_feat_set+0xf0/0xf0 [ 1048.212032] [] ? sched_feat_set+0xf0/0xf0 [ 1048.212050] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.212074] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.212092] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.212115] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.212117] [] ? kthread+0xd1/0xe0 [ 1048.212118] [] ? kthread+0xd1/0xe0 [ 1048.212120] [] ? insert_kthread_work+0x40/0x40 [ 1048.212121] [] ? insert_kthread_work+0x40/0x40 [ 1048.212123] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.212125] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.212126] [] ? insert_kthread_work+0x40/0x40 [ 1048.212127] [] ? insert_kthread_work+0x40/0x40 [ 1048.212128] Task dump for CPU 30: [ 1048.212128] Task dump for CPU 14: [ 1048.212129] mdt01_018 R [ 1048.212129] mdt01_016 R [ 1048.212130] running task [ 1048.212130] running task [ 1048.212131] 0 17592 2 0x00000088 [ 1048.212132] 0 17554 2 0x00000088 [ 1048.212132] Call Trace: [ 1048.212132] Call Trace: [ 1048.212134] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.212135] [ 1048.212136] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.212138] [] sched_show_task+0xbf/0x120 [ 1048.212139] [] ? _raw_spin_lock+0x30/0x40 [ 1048.212141] [] dump_cpu_task+0x39/0x70 [ 1048.212146] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.212148] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 1048.212154] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.212156] [] rcu_check_callbacks+0x482/0x770 [ 1048.212157] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.212159] [] update_process_times+0x46/0x80 [ 1048.212165] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.212167] [] tick_sched_handle+0x30/0x70 [ 1048.212184] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.212186] [] tick_sched_timer+0x39/0x80 [ 1048.212201] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.212203] [] __hrtimer_run_queues+0x13e/0x2f0 [ 1048.212204] [] ? check_preempt_curr+0x80/0xa0 [ 1048.212206] [] ? tick_sched_do_timer+0x50/0x50 [ 1048.212207] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.212208] [] hrtimer_interrupt+0xb9/0x1f0 [ 1048.212231] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.212232] [] local_apic_timer_interrupt+0x3b/0x60 [ 1048.212253] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.212255] [] smp_apic_timer_interrupt+0x43/0x60 [ 1048.212256] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.212258] [] apic_timer_interrupt+0x16a/0x170 [ 1048.212276] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.212277] [ 1048.212294] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.212296] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.212297] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.212299] [] queued_spin_lock_slowpath+0xb/0xf [ 1048.212300] [] ? sched_feat_set+0xf0/0xf0 [ 1048.212302] [] _raw_spin_lock+0x30/0x40 [ 1048.212320] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.212326] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.212343] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.212351] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.212352] [] ? kthread+0xd1/0xe0 [ 1048.212354] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.212355] [] ? insert_kthread_work+0x40/0x40 [ 1048.212363] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.212364] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.212388] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.212389] [] ? insert_kthread_work+0x40/0x40 [ 1048.212412] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.212414] [] ? check_preempt_curr+0x80/0xa0 [ 1048.212437] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.212460] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.212490] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.212518] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.212520] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.212547] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.212572] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.212573] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.212574] [] ? sched_feat_set+0xf0/0xf0 [ 1048.212599] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.212623] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.212625] [] kthread+0xd1/0xe0 [ 1048.212627] [] ? insert_kthread_work+0x40/0x40 [ 1048.212629] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1048.212630] [] ? insert_kthread_work+0x40/0x40 [ 1048.212631] Task dump for CPU 15: [ 1048.212632] mdt01_007 R [ 1048.212632] running task [ 1048.212633] 0 17310 2 0x00000088 [ 1048.212633] Call Trace: [ 1048.212635] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.212637] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.212639] [] ? _raw_spin_lock+0x30/0x40 [ 1048.212645] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.212653] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.212655] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.212663] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.212686] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.212710] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.212712] [] ? check_preempt_curr+0x80/0xa0 [ 1048.212735] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.212758] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.212789] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.212817] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.212819] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.212844] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.212869] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.212870] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.212871] [] ? sched_feat_set+0xf0/0xf0 [ 1048.212896] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.212920] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.212922] [] ? kthread+0xd1/0xe0 [ 1048.212924] [] ? insert_kthread_work+0x40/0x40 [ 1048.212926] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.212927] [] ? insert_kthread_work+0x40/0x40 [ 1048.212928] Task dump for CPU 20: [ 1048.212929] Task dump for CPU 2: [ 1048.212930] ll_mgs_0013 R [ 1048.212930] mdt00_013 R [ 1048.212931] running task [ 1048.212931] running task [ 1048.212932] 0 10822 2 0x00000088 [ 1048.212933] 0 17594 2 0x00000088 [ 1048.212934] Call Trace: [ 1048.212934] Call Trace: [ 1048.212936] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.212939] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.212941] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.212942] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.212944] [] ? _raw_spin_lock+0x30/0x40 [ 1048.212945] [] ? _raw_spin_lock+0x30/0x40 [ 1048.212952] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 1048.212956] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.212962] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1048.212968] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.212976] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1048.212977] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.212979] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.212985] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.212992] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.213011] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.213035] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.213050] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.213072] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.213090] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.213092] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.213093] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.213095] [] ? check_preempt_curr+0x80/0xa0 [ 1048.213096] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.213098] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.213099] [] ? check_preempt_curr+0x80/0xa0 [ 1048.213129] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.213130] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.213158] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.213181] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.213183] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.213203] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.213229] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.213230] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.213255] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.213275] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.213276] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.213295] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.213296] [] ? sched_feat_set+0xf0/0xf0 [ 1048.213298] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.213323] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.213324] [] ? sched_feat_set+0xf0/0xf0 [ 1048.213325] [] ? __switch_to+0xce/0x5a0 [ 1048.213343] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.213367] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.213369] [] ? __switch_to+0xce/0x5a0 [ 1048.213371] [] ? kthread+0xd1/0xe0 [ 1048.213388] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.213390] [] ? insert_kthread_work+0x40/0x40 [ 1048.213392] [] ? kthread+0xd1/0xe0 [ 1048.213393] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.213395] [] ? insert_kthread_work+0x40/0x40 [ 1048.213397] [] ? insert_kthread_work+0x40/0x40 [ 1048.213398] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.213398] Task dump for CPU 23: [ 1048.213400] [] ? insert_kthread_work+0x40/0x40 [ 1048.213400] ll_mgs_0014 R [ 1048.213401] Task dump for CPU 8: [ 1048.213401] running task [ 1048.213401] mdt01_022 R [ 1048.213403] 0 10826 2 0x00000088 [ 1048.213403] running task [ 1048.213403] Call Trace: [ 1048.213404] 0 17601 2 0x00000088 [ 1048.213406] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.213406] Call Trace: [ 1048.213408] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.213410] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.213411] [] ? _raw_spin_lock+0x30/0x40 [ 1048.213413] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.213419] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.213420] [] ? _raw_spin_lock+0x30/0x40 [ 1048.213428] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.213433] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.213435] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.213441] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.213448] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.213450] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.213474] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.213479] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.213504] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.213521] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.213523] [] ? check_preempt_curr+0x80/0xa0 [ 1048.213541] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.213565] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.213583] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.213606] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.213623] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.213653] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.213669] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.213697] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.213719] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.213721] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.213743] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.213769] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.213770] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.213796] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.213815] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.213816] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.213834] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.213835] [] ? sched_feat_set+0xf0/0xf0 [ 1048.213837] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.213861] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.213862] [] ? sched_feat_set+0xf0/0xf0 [ 1048.213864] [] ? __switch_to+0xce/0x5a0 [ 1048.213882] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.213907] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.213924] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.213926] [] ? kthread+0xd1/0xe0 [ 1048.213927] [] ? kthread+0xd1/0xe0 [ 1048.213929] [] ? insert_kthread_work+0x40/0x40 [ 1048.213930] [] ? insert_kthread_work+0x40/0x40 [ 1048.213932] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.213933] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.213935] [] ? insert_kthread_work+0x40/0x40 [ 1048.213936] [] ? insert_kthread_work+0x40/0x40 [ 1048.213937] Task dump for CPU 29: [ 1048.213937] Task dump for CPU 13: [ 1048.213938] mdt01_017 R [ 1048.213938] mdt01_003 R [ 1048.213939] running task [ 1048.213939] running task [ 1048.213940] 0 17555 2 0x00000088 [ 1048.213940] 0 10825 2 0x00000088 [ 1048.213941] Call Trace: [ 1048.213941] Call Trace: [ 1048.213943] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.213945] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.213947] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.213948] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.213950] [] ? _raw_spin_lock+0x30/0x40 [ 1048.213951] [] ? _raw_spin_lock+0x30/0x40 [ 1048.213957] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.213962] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.213970] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.213976] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.213978] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.213979] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.213987] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.213993] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.214017] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.214034] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.214059] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.214074] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.214076] [] ? check_preempt_curr+0x80/0xa0 [ 1048.214077] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.214101] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.214102] [] ? check_preempt_curr+0x80/0xa0 [ 1048.214125] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.214126] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.214157] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.214179] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.214207] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.214227] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.214229] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.214230] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.214256] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.214275] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.214300] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.214318] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.214320] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.214321] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.214323] [] ? sched_feat_set+0xf0/0xf0 [ 1048.214324] [] ? sched_feat_set+0xf0/0xf0 [ 1048.214348] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.214366] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.214390] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.214408] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.214410] [] ? kthread+0xd1/0xe0 [ 1048.214411] [] ? kthread+0xd1/0xe0 [ 1048.214413] [] ? insert_kthread_work+0x40/0x40 [ 1048.214414] [] ? insert_kthread_work+0x40/0x40 [ 1048.214416] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.214417] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.214419] [] ? insert_kthread_work+0x40/0x40 [ 1048.214420] [] ? insert_kthread_work+0x40/0x40 [ 1048.214421] Task dump for CPU 30: [ 1048.214421] Task dump for CPU 14: [ 1048.214422] mdt01_018 R [ 1048.214422] mdt01_016 R [ 1048.214423] running task [ 1048.214423] running task [ 1048.214424] 0 17592 2 0x00000088 [ 1048.214425] 0 17554 2 0x00000088 [ 1048.214425] Call Trace: [ 1048.214426] Call Trace: [ 1048.214427] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.214429] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.214431] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.214433] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.214434] [] ? _raw_spin_lock+0x30/0x40 [ 1048.214436] [] ? _raw_spin_lock+0x30/0x40 [ 1048.214442] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.214446] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.214454] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.214460] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.214462] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.214463] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.214471] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.214477] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.214500] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.214518] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.214540] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.214557] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.214559] [] ? check_preempt_curr+0x80/0xa0 [ 1048.214560] [] ? check_preempt_curr+0x80/0xa0 [ 1048.214562] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.214579] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.214608] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.214625] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.214653] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.214675] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.214677] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.214697] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.214724] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.214725] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.214751] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.214770] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.214771] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.214789] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.214791] [] ? sched_feat_set+0xf0/0xf0 [ 1048.214792] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.214817] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.214818] [] ? sched_feat_set+0xf0/0xf0 [ 1048.214841] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.214859] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.214861] [] ? kthread+0xd1/0xe0 [ 1048.214878] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.214880] [] ? insert_kthread_work+0x40/0x40 [ 1048.214882] [] ? kthread+0xd1/0xe0 [ 1048.214884] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.214885] [] ? insert_kthread_work+0x40/0x40 [ 1048.214887] [] ? insert_kthread_work+0x40/0x40 [ 1048.214888] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.214889] [] ? insert_kthread_work+0x40/0x40 [ 1048.214890] Task dump for CPU 15: [ 1048.214890] mdt01_007 R [ 1048.214890] running task [ 1048.214891] 0 17310 2 0x00000088 [ 1048.214891] Call Trace: [ 1048.214892] [ 1048.214894] [] sched_show_task+0xbf/0x120 [ 1048.214895] [] dump_cpu_task+0x39/0x70 [ 1048.214897] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 1048.214898] [] rcu_check_callbacks+0x482/0x770 [ 1048.214900] [] update_process_times+0x46/0x80 [ 1048.214902] [] tick_sched_handle+0x30/0x70 [ 1048.214903] [] tick_sched_timer+0x39/0x80 [ 1048.214904] [] __hrtimer_run_queues+0x13e/0x2f0 [ 1048.214906] [] ? tick_sched_do_timer+0x50/0x50 [ 1048.214907] [] hrtimer_interrupt+0xb9/0x1f0 [ 1048.214908] [] local_apic_timer_interrupt+0x3b/0x60 [ 1048.214910] [] smp_apic_timer_interrupt+0x43/0x60 [ 1048.214911] [] apic_timer_interrupt+0x16a/0x170 [ 1048.214911] [ 1048.214913] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.214914] [] queued_spin_lock_slowpath+0xb/0xf [ 1048.214915] [] _raw_spin_lock+0x30/0x40 [ 1048.214920] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.214927] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.214928] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.214934] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.214951] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.214969] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.214970] [] ? check_preempt_curr+0x80/0xa0 [ 1048.214987] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.215004] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.215026] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.215047] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.215048] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.215067] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.215085] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.215087] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.215087] [] ? sched_feat_set+0xf0/0xf0 [ 1048.215105] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.215123] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.215125] [] kthread+0xd1/0xe0 [ 1048.215126] [] ? insert_kthread_work+0x40/0x40 [ 1048.215127] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1048.215129] [] ? insert_kthread_work+0x40/0x40 [ 1048.215129] Task dump for CPU 20: [ 1048.215130] Task dump for CPU 2: [ 1048.215130] ll_mgs_0013 R [ 1048.215131] mdt00_013 R [ 1048.215131] running task [ 1048.215132] running task [ 1048.215132] 0 10822 2 0x00000088 [ 1048.215133] 0 17594 2 0x00000088 [ 1048.215134] Call Trace: [ 1048.215134] Call Trace: [ 1048.215135] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.215137] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.215139] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.215141] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.215143] [] ? _raw_spin_lock+0x30/0x40 [ 1048.215144] [] ? _raw_spin_lock+0x30/0x40 [ 1048.215149] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 1048.215155] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.215159] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1048.215167] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.215173] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1048.215175] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.215176] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.215184] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.215189] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.215213] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.215230] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.215252] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.215268] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.215291] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.215293] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.215294] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.215296] [] ? check_preempt_curr+0x80/0xa0 [ 1048.215298] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.215299] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.215300] [] ? check_preempt_curr+0x80/0xa0 [ 1048.215322] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.215324] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.215344] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.215374] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.215376] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.215403] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.215422] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.215423] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.215441] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.215466] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.215467] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.215491] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.215492] [] ? sched_feat_set+0xf0/0xf0 [ 1048.215494] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.215519] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.215520] [] ? sched_feat_set+0xf0/0xf0 [ 1048.215522] [] ? __switch_to+0xce/0x5a0 [ 1048.215546] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.215571] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.215573] [] ? __switch_to+0xce/0x5a0 [ 1048.215574] [] ? kthread+0xd1/0xe0 [ 1048.215598] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.215599] [] ? insert_kthread_work+0x40/0x40 [ 1048.215601] [] ? kthread+0xd1/0xe0 [ 1048.215603] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.215604] [] ? insert_kthread_work+0x40/0x40 [ 1048.215606] [] ? insert_kthread_work+0x40/0x40 [ 1048.215607] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.215608] Task dump for CPU 23: [ 1048.215609] [] ? insert_kthread_work+0x40/0x40 [ 1048.215610] ll_mgs_0014 R [ 1048.215610] running task [ 1048.215611] Task dump for CPU 8: [ 1048.215611] 0 10826 2 0x00000088 [ 1048.215612] mdt01_022 R [ 1048.215612] Call Trace: [ 1048.215613] running task [ 1048.215614] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.215615] 0 17601 2 0x00000088 [ 1048.215617] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.215617] Call Trace: [ 1048.215619] [] ? _raw_spin_lock+0x30/0x40 [ 1048.215620] [] ? native_queued_spin_lock_slowpath+0x120/0x200 [ 1048.215625] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.215627] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.215634] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.215636] [] ? _raw_spin_lock+0x30/0x40 [ 1048.215637] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.215643] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.215650] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.215658] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.215682] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.215683] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.215707] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.215714] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.215716] [] ? check_preempt_curr+0x80/0xa0 [ 1048.215739] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.215757] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.215780] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.215797] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.215821] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.215844] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.215867] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.215888] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.215912] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.215914] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.215943] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.215962] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.215990] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.216008] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.216010] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.216011] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.216037] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.216038] [] ? sched_feat_set+0xf0/0xf0 [ 1048.216062] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.216080] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.216081] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.216083] [] ? __switch_to+0xce/0x5a0 [ 1048.216084] [] ? sched_feat_set+0xf0/0xf0 [ 1048.216101] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.216125] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.216127] [] ? kthread+0xd1/0xe0 [ 1048.216150] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.216152] [] ? insert_kthread_work+0x40/0x40 [ 1048.216154] [] ? kthread+0xd1/0xe0 [ 1048.216155] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.216157] [] ? insert_kthread_work+0x40/0x40 [ 1048.216158] [] ? insert_kthread_work+0x40/0x40 [ 1048.216160] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.216160] Task dump for CPU 29: [ 1048.216162] [] ? insert_kthread_work+0x40/0x40 [ 1048.216162] mdt01_017 R [ 1048.216163] Task dump for CPU 13: [ 1048.216163] running task [ 1048.216163] mdt01_003 R [ 1048.216164] 0 17555 2 0x00000088 [ 1048.216165] running task [ 1048.216165] Call Trace: [ 1048.216166] 0 10825 2 0x00000088 [ 1048.216167] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.216168] Call Trace: [ 1048.216169] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.216171] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.216173] [] ? _raw_spin_lock+0x30/0x40 [ 1048.216175] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.216179] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.216181] [] ? _raw_spin_lock+0x30/0x40 [ 1048.216187] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.216193] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.216194] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.216202] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.216208] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.216210] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.216227] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.216234] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.216252] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.216277] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.216278] [] ? check_preempt_curr+0x80/0xa0 [ 1048.216312] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.216329] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.216331] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.216347] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.216348] [] ? check_preempt_curr+0x80/0xa0 [ 1048.216371] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.216372] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.216393] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.216432] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.216434] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.216472] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.216491] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.216493] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.216511] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.216548] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.216550] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.216587] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.216588] [] ? sched_feat_set+0xf0/0xf0 [ 1048.216590] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.216608] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.216609] [] ? sched_feat_set+0xf0/0xf0 [ 1048.216627] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.216664] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.216666] [] ? kthread+0xd1/0xe0 [ 1048.216703] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.216704] [] ? insert_kthread_work+0x40/0x40 [ 1048.216706] [] ? kthread+0xd1/0xe0 [ 1048.216708] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.216709] [] ? insert_kthread_work+0x40/0x40 [ 1048.216711] [] ? insert_kthread_work+0x40/0x40 [ 1048.216712] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.216713] Task dump for CPU 30: [ 1048.216715] [] ? insert_kthread_work+0x40/0x40 [ 1048.216715] mdt01_018 R [ 1048.216716] Task dump for CPU 14: [ 1048.216716] running task [ 1048.216717] mdt01_016 R [ 1048.216717] 0 17592 2 0x00000088 [ 1048.216718] running task [ 1048.216718] Call Trace: [ 1048.216719] 0 17554 2 0x00000088 [ 1048.216720] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.216721] Call Trace: [ 1048.216722] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.216724] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.216726] [] ? _raw_spin_lock+0x30/0x40 [ 1048.216728] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.216734] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.216736] [] ? _raw_spin_lock+0x30/0x40 [ 1048.216741] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.216749] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.216750] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.216760] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.216765] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.216767] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.216784] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.216794] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.216809] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.216845] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.216846] [] ? check_preempt_curr+0x80/0xa0 [ 1048.216882] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.216883] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.216885] [] ? check_preempt_curr+0x80/0xa0 [ 1048.216908] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.216943] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.216964] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.217000] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.217001] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.217041] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.217060] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.217098] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.217116] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.217118] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.217119] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.217156] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.217157] [] ? sched_feat_set+0xf0/0xf0 [ 1048.217194] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.217212] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.217214] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.217231] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.217233] [] ? sched_feat_set+0xf0/0xf0 [ 1048.217234] [] ? kthread+0xd1/0xe0 [ 1048.217272] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.217273] [] ? insert_kthread_work+0x40/0x40 [ 1048.217310] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.217312] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.217314] [] ? kthread+0xd1/0xe0 [ 1048.217315] [] ? insert_kthread_work+0x40/0x40 [ 1048.217317] [] ? insert_kthread_work+0x40/0x40 [ 1048.217318] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.217320] [] ? insert_kthread_work+0x40/0x40 [ 1048.217321] Task dump for CPU 15: [ 1048.217321] mdt01_007 R [ 1048.217322] running task [ 1048.217323] 0 17310 2 0x00000088 [ 1048.217323] Call Trace: [ 1048.217325] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.217327] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.217329] [] ? _raw_spin_lock+0x30/0x40 [ 1048.217336] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.217346] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.217348] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.217358] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.217394] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.217430] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.217432] [] ? check_preempt_curr+0x80/0xa0 [ 1048.217468] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.217504] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.217544] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.217583] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.217584] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.217622] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.217660] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.217662] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.217663] [] ? sched_feat_set+0xf0/0xf0 [ 1048.217701] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.217739] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.217741] [] ? kthread+0xd1/0xe0 [ 1048.217743] [] ? insert_kthread_work+0x40/0x40 [ 1048.217745] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.217747] [] ? insert_kthread_work+0x40/0x40 [ 1048.217747] Task dump for CPU 20: [ 1048.217748] Task dump for CPU 2: [ 1048.217749] ll_mgs_0013 R [ 1048.217750] mdt00_013 R [ 1048.217751] running task [ 1048.217752] running task [ 1048.217753] 0 10822 2 0x00000088 [ 1048.217755] 0 17594 2 0x00000088 [ 1048.217756] Call Trace: [ 1048.217756] Call Trace: [ 1048.217759] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.217760] [ 1048.217763] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.217766] [] sched_show_task+0xbf/0x120 [ 1048.217768] [] ? _raw_spin_lock+0x30/0x40 [ 1048.217770] [] dump_cpu_task+0x39/0x70 [ 1048.217778] [] ? cfs_percpt_lock+0x58/0x110 [libcfs] [ 1048.217781] [] rcu_dump_cpu_stacks+0x90/0xd0 [ 1048.217788] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1048.217791] [] rcu_check_callbacks+0x482/0x770 [ 1048.217801] [] ? lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1048.217804] [] update_process_times+0x46/0x80 [ 1048.217807] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.217810] [] tick_sched_handle+0x30/0x70 [ 1048.217820] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.217822] [] tick_sched_timer+0x39/0x80 [ 1048.217859] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.217861] [] __hrtimer_run_queues+0x13e/0x2f0 [ 1048.217895] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.217897] [] ? tick_sched_do_timer+0x50/0x50 [ 1048.217900] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.217901] [] hrtimer_interrupt+0xb9/0x1f0 [ 1048.217904] [] ? check_preempt_curr+0x80/0xa0 [ 1048.217906] [] local_apic_timer_interrupt+0x3b/0x60 [ 1048.217908] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.217911] [] smp_apic_timer_interrupt+0x43/0x60 [ 1048.217951] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.217952] [] apic_timer_interrupt+0x16a/0x170 [ 1048.217992] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.217993] [ 1048.217995] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.217997] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.218034] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.218037] [] queued_spin_lock_slowpath+0xb/0xf [ 1048.218075] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.218076] [] _raw_spin_lock+0x30/0x40 [ 1048.218079] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.218084] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.218086] [] ? sched_feat_set+0xf0/0xf0 [ 1048.218092] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.218130] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.218133] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.218135] [] ? __switch_to+0xce/0x5a0 [ 1048.218141] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.218179] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.218199] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.218202] [] ? kthread+0xd1/0xe0 [ 1048.218218] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.218221] [] ? insert_kthread_work+0x40/0x40 [ 1048.218239] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.218241] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.218243] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.218245] [] ? insert_kthread_work+0x40/0x40 [ 1048.218247] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.218248] Task dump for CPU 23: [ 1048.218250] [] ? check_preempt_curr+0x80/0xa0 [ 1048.218251] ll_mgs_0014 R [ 1048.218253] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.218254] running task [ 1048.218277] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.218279] 0 10826 2 0x00000088 [ 1048.218300] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.218301] Call Trace: [ 1048.218303] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.218306] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.218326] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.218329] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.218347] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.218350] [] ? _raw_spin_lock+0x30/0x40 [ 1048.218351] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.218359] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.218361] [] ? sched_feat_set+0xf0/0xf0 [ 1048.218371] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.218390] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.218392] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.218394] [] ? __switch_to+0xce/0x5a0 [ 1048.218404] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.218422] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.218458] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.218460] [] kthread+0xd1/0xe0 [ 1048.218497] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.218499] [] ? insert_kthread_work+0x40/0x40 [ 1048.218501] [] ? check_preempt_curr+0x80/0xa0 [ 1048.218503] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1048.218539] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.218541] [] ? insert_kthread_work+0x40/0x40 [ 1048.218577] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.218578] Task dump for CPU 8: [ 1048.218618] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.218620] mdt01_022 R [ 1048.218659] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.218659] running task [ 1048.218662] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.218663] 0 17601 2 0x00000088 [ 1048.218701] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.218702] Call Trace: [ 1048.218740] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.218743] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.218745] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.218747] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.218749] [] ? sched_feat_set+0xf0/0xf0 [ 1048.218751] [] ? _raw_spin_lock+0x30/0x40 [ 1048.218788] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.218794] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.218797] [] ? __switch_to+0xce/0x5a0 [ 1048.218804] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.218842] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.218844] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.218846] [] ? kthread+0xd1/0xe0 [ 1048.218852] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.218854] [] ? insert_kthread_work+0x40/0x40 [ 1048.218872] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.218875] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.218892] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.218895] [] ? insert_kthread_work+0x40/0x40 [ 1048.218913] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.218914] Task dump for CPU 29: [ 1048.218932] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.218933] mdt01_017 R [ 1048.218950] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.218951] running task [ 1048.218973] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.218975] 0 17555 2 0x00000088 [ 1048.218996] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.218997] Call Trace: [ 1048.218999] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.219002] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.219022] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.219025] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.219043] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.219046] [] ? _raw_spin_lock+0x30/0x40 [ 1048.219047] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.219055] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.219057] [] ? sched_feat_set+0xf0/0xf0 [ 1048.219067] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.219086] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.219088] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.219106] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.219116] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.219118] [] ? kthread+0xd1/0xe0 [ 1048.219155] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.219157] [] ? insert_kthread_work+0x40/0x40 [ 1048.219193] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.219195] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.219197] [] ? check_preempt_curr+0x80/0xa0 [ 1048.219199] [] ? insert_kthread_work+0x40/0x40 [ 1048.219235] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.219236] Task dump for CPU 13: [ 1048.219273] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.219274] mdt01_003 R [ 1048.219314] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.219314] running task [ 1048.219355] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.219356] 0 10825 2 0x00000088 [ 1048.219357] Call Trace: [ 1048.219359] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.219361] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.219362] [] ? _raw_spin_lock+0x30/0x40 [ 1048.219367] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.219373] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.219375] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.219381] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.219398] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.219413] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.219415] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.219417] [] ? check_preempt_curr+0x80/0xa0 [ 1048.219418] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.219420] { 20} (t=240015 jiffies g=29074 c=29073 q=216843) [ 1048.219442] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.219463] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.219464] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.219483] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.219501] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.219503] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.219503] [] ? sched_feat_set+0xf0/0xf0 [ 1048.219521] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.219539] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.219540] [] ? kthread+0xd1/0xe0 [ 1048.219542] [] ? insert_kthread_work+0x40/0x40 [ 1048.219543] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.219544] [] ? insert_kthread_work+0x40/0x40 [ 1048.219545] Task dump for CPU 14: [ 1048.219546] mdt01_016 R running task 0 17554 2 0x00000088 [ 1048.219547] Call Trace: [ 1048.219548] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.219550] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.219551] [] ? _raw_spin_lock+0x30/0x40 [ 1048.219556] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.219562] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.219565] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.219571] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.219588] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.219606] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.219607] [] ? check_preempt_curr+0x80/0xa0 [ 1048.219624] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.219641] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.219663] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.219684] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.219685] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.219704] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.219727] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.219729] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.219730] [] ? sched_feat_set+0xf0/0xf0 [ 1048.219749] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.219767] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.219769] [] ? kthread+0xd1/0xe0 [ 1048.219771] [] ? insert_kthread_work+0x40/0x40 [ 1048.219773] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.219774] [] ? insert_kthread_work+0x40/0x40 [ 1048.219775] Task dump for CPU 15: [ 1048.219776] mdt01_007 R running task 0 17310 2 0x00000088 [ 1048.219776] Call Trace: [ 1048.219778] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.219780] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.219781] [] ? _raw_spin_lock+0x30/0x40 [ 1048.219786] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.219792] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.219793] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.219799] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.219816] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.219833] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.219834] [] ? check_preempt_curr+0x80/0xa0 [ 1048.219851] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.219867] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.219889] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.219910] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.219911] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.219930] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.219948] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.219949] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.219950] [] ? sched_feat_set+0xf0/0xf0 [ 1048.219968] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.219985] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.219987] [] ? kthread+0xd1/0xe0 [ 1048.219988] [] ? insert_kthread_work+0x40/0x40 [ 1048.219990] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.219991] [] ? insert_kthread_work+0x40/0x40 [ 1048.219992] Task dump for CPU 2: [ 1048.219994] mdt00_013 R running task 0 17594 2 0x00000088 [ 1048.219994] Call Trace: [ 1048.219997] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.219999] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.220000] [] ? _raw_spin_lock+0x30/0x40 [ 1048.220005] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.220012] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.220013] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.220019] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.220037] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.220053] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.220071] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.220072] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.220074] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.220075] [] ? check_preempt_curr+0x80/0xa0 [ 1048.220076] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.220099] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.220121] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.220122] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.220141] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.220160] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.220161] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.220162] [] ? sched_feat_set+0xf0/0xf0 [ 1048.220180] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.220181] [] ? __switch_to+0xce/0x5a0 [ 1048.220199] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.220201] [] ? kthread+0xd1/0xe0 [ 1048.220202] [] ? insert_kthread_work+0x40/0x40 [ 1048.220203] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.220205] [] ? insert_kthread_work+0x40/0x40 [ 1048.220205] Task dump for CPU 8: [ 1048.220207] mdt01_022 R running task 0 17601 2 0x00000088 [ 1048.220207] Call Trace: [ 1048.220209] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.220211] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.220212] [] ? _raw_spin_lock+0x30/0x40 [ 1048.220217] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.220223] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.220225] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.220230] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.220248] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.220265] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.220284] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.220301] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.220318] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.220340] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.220361] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.220362] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.220381] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.220399] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.220400] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.220401] [] ? sched_feat_set+0xf0/0xf0 [ 1048.220419] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.220437] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.220438] [] ? kthread+0xd1/0xe0 [ 1048.220440] [] ? insert_kthread_work+0x40/0x40 [ 1048.220441] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.220442] [] ? insert_kthread_work+0x40/0x40 [ 1048.220443] Task dump for CPU 13: [ 1048.220444] mdt01_003 R running task 0 10825 2 0x00000088 [ 1048.220444] Call Trace: [ 1048.220446] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.220448] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.220449] [] ? _raw_spin_lock+0x30/0x40 [ 1048.220454] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.220460] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.220461] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.220467] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.220484] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.220499] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.220501] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.220502] [] ? check_preempt_curr+0x80/0xa0 [ 1048.220503] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.220526] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.220547] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.220548] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.220567] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.220584] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.220586] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.220587] [] ? sched_feat_set+0xf0/0xf0 [ 1048.220604] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.220621] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.220623] [] ? kthread+0xd1/0xe0 [ 1048.220624] [] ? insert_kthread_work+0x40/0x40 [ 1048.220626] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.220627] [] ? insert_kthread_work+0x40/0x40 [ 1048.220627] Task dump for CPU 14: [ 1048.220629] mdt01_016 R running task 0 17554 2 0x00000088 [ 1048.220629] Call Trace: [ 1048.220630] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.220632] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.220634] [] ? _raw_spin_lock+0x30/0x40 [ 1048.220638] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.220644] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.220646] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.220651] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.220669] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.220686] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.220688] [] ? check_preempt_curr+0x80/0xa0 [ 1048.220705] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.220721] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.220746] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.220766] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.220767] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.220786] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.220803] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.220804] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.220805] [] ? sched_feat_set+0xf0/0xf0 [ 1048.220823] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.220840] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.220842] [] ? kthread+0xd1/0xe0 [ 1048.220843] [] ? insert_kthread_work+0x40/0x40 [ 1048.220845] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.220846] [] ? insert_kthread_work+0x40/0x40 [ 1048.220846] Task dump for CPU 15: [ 1048.220848] mdt01_007 R running task 0 17310 2 0x00000088 [ 1048.220848] Call Trace: [ 1048.220849] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.220851] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.220852] [] ? _raw_spin_lock+0x30/0x40 [ 1048.220857] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.220863] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.220865] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.220870] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.220888] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.220905] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.220906] [] ? check_preempt_curr+0x80/0xa0 [ 1048.220923] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.220940] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.220962] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.220982] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.220984] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.221009] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.221034] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.221035] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.221036] [] ? sched_feat_set+0xf0/0xf0 [ 1048.221062] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.221087] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.221089] [] ? kthread+0xd1/0xe0 [ 1048.221090] [] ? insert_kthread_work+0x40/0x40 [ 1048.221091] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.221093] [] ? insert_kthread_work+0x40/0x40 [ 1048.221094] Task dump for CPU 2: [ 1048.221096] mdt00_013 R running task 0 17594 2 0x00000088 [ 1048.221096] Call Trace: [ 1048.221099] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.221101] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.221102] [] ? _raw_spin_lock+0x30/0x40 [ 1048.221109] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.221117] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.221119] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.221127] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.221150] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.221173] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.221197] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.221199] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.221201] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.221202] [] ? check_preempt_curr+0x80/0xa0 [ 1048.221204] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.221234] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.221262] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.221263] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.221289] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.221313] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.221314] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.221316] [] ? sched_feat_set+0xf0/0xf0 [ 1048.221340] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.221342] [] ? __switch_to+0xce/0x5a0 [ 1048.221365] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.221367] [] ? kthread+0xd1/0xe0 [ 1048.221369] [] ? insert_kthread_work+0x40/0x40 [ 1048.221371] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.221372] [] ? insert_kthread_work+0x40/0x40 [ 1048.221373] Task dump for CPU 8: [ 1048.221374] mdt01_022 R running task 0 17601 2 0x00000088 [ 1048.221375] Call Trace: [ 1048.221377] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.221379] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.221380] [] ? _raw_spin_lock+0x30/0x40 [ 1048.221387] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.221395] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.221397] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.221405] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.221428] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.221452] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.221476] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.221499] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.221522] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.221552] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.221579] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.221581] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.221608] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.221632] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.221633] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.221634] [] ? sched_feat_set+0xf0/0xf0 [ 1048.221659] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.221682] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.221684] [] ? kthread+0xd1/0xe0 [ 1048.221686] [] ? insert_kthread_work+0x40/0x40 [ 1048.221688] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.221689] [] ? insert_kthread_work+0x40/0x40 [ 1048.221690] Task dump for CPU 13: [ 1048.221692] mdt01_003 R running task 0 10825 2 0x00000088 [ 1048.221692] Call Trace: [ 1048.221694] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.221696] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.221697] [] ? _raw_spin_lock+0x30/0x40 [ 1048.221704] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.221712] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.221714] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.221721] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.221745] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.221766] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.221768] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.221770] [] ? check_preempt_curr+0x80/0xa0 [ 1048.221771] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.221801] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.221829] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.221831] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.221857] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.221881] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.221883] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.221884] [] ? sched_feat_set+0xf0/0xf0 [ 1048.221908] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.221932] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.221934] [] ? kthread+0xd1/0xe0 [ 1048.221936] [] ? insert_kthread_work+0x40/0x40 [ 1048.221937] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.221939] [] ? insert_kthread_work+0x40/0x40 [ 1048.221940] Task dump for CPU 14: [ 1048.221941] mdt01_016 R running task 0 17554 2 0x00000088 [ 1048.221941] Call Trace: [ 1048.221943] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.221946] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.221947] [] ? _raw_spin_lock+0x30/0x40 [ 1048.221954] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.221962] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.221964] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.221971] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.221995] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.222018] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.222020] [] ? check_preempt_curr+0x80/0xa0 [ 1048.222043] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.222066] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.222096] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.222124] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.222126] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.222151] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.222175] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.222177] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.222178] [] ? sched_feat_set+0xf0/0xf0 [ 1048.222202] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.222226] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.222228] [] ? kthread+0xd1/0xe0 [ 1048.222230] [] ? insert_kthread_work+0x40/0x40 [ 1048.222232] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.222233] [] ? insert_kthread_work+0x40/0x40 [ 1048.222234] Task dump for CPU 15: [ 1048.222236] mdt01_007 R running task 0 17310 2 0x00000088 [ 1048.222236] Call Trace: [ 1048.222238] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.222240] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.222242] [] ? _raw_spin_lock+0x30/0x40 [ 1048.222248] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.222256] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.222258] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.222265] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.222288] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.222312] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.222314] [] ? check_preempt_curr+0x80/0xa0 [ 1048.222337] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.222360] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.222390] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.222417] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.222419] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.222444] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.222468] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.222470] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.222471] [] ? sched_feat_set+0xf0/0xf0 [ 1048.222496] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.222520] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.222521] [] ? kthread+0xd1/0xe0 [ 1048.222523] [] ? insert_kthread_work+0x40/0x40 [ 1048.222525] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.222527] [] ? insert_kthread_work+0x40/0x40 [ 1048.222531] Task dump for CPU 2: [ 1048.222533] mdt00_013 R running task 0 17594 2 0x00000088 [ 1048.222534] Call Trace: [ 1048.222537] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.222539] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.222541] [] ? _raw_spin_lock+0x30/0x40 [ 1048.222548] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.222556] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.222558] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.222566] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.222591] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.222614] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.222639] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1048.222641] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.222643] [] ? sched_clock_cpu+0xa5/0xe0 [ 1048.222645] [] ? check_preempt_curr+0x80/0xa0 [ 1048.222647] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.222678] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.222707] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.222709] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.222736] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.222762] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.222764] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.222765] [] ? sched_feat_set+0xf0/0xf0 [ 1048.222791] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.222793] [] ? __switch_to+0xce/0x5a0 [ 1048.222818] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.222820] [] ? kthread+0xd1/0xe0 [ 1048.222822] [] ? insert_kthread_work+0x40/0x40 [ 1048.222824] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.222826] [] ? insert_kthread_work+0x40/0x40 [ 1048.222826] Task dump for CPU 8: [ 1048.222828] mdt01_022 R running task 0 17601 2 0x00000088 [ 1048.222829] Call Trace: [ 1048.222831] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.222833] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.222835] [] ? _raw_spin_lock+0x30/0x40 [ 1048.222841] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.222850] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.222852] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.222859] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.222884] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.222908] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.222934] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1048.222958] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.222983] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.223013] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.223041] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.223043] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.223069] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.223095] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.223096] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.223098] [] ? sched_feat_set+0xf0/0xf0 [ 1048.223122] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.223147] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.223149] [] ? kthread+0xd1/0xe0 [ 1048.223151] [] ? insert_kthread_work+0x40/0x40 [ 1048.223153] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.223155] [] ? insert_kthread_work+0x40/0x40 [ 1048.223156] Task dump for CPU 13: [ 1048.223157] mdt01_003 R running task 0 10825 2 0x00000088 [ 1048.223158] Call Trace: [ 1048.223160] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1048.223162] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.223164] [] ? _raw_spin_lock+0x30/0x40 [ 1048.223170] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.223178] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.223181] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.223188] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.223213] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.223235] [] ? target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1048.223237] [] ? enqueue_task_fair+0x208/0x6c0 [ 1048.223239] [] ? check_preempt_curr+0x80/0xa0 [ 1048.223241] [] ? ttwu_do_wakeup+0x19/0x100 [ 1048.223271] [] ? tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1048.223299] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.223301] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.223326] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.223351] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.223353] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.223354] [] ? sched_feat_set+0xf0/0xf0 [ 1048.223379] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.223404] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.223406] [] ? kthread+0xd1/0xe0 [ 1048.223408] [] ? insert_kthread_work+0x40/0x40 [ 1048.223410] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.223411] [] ? insert_kthread_work+0x40/0x40 [ 1048.223412] Task dump for CPU 14: [ 1048.223414] mdt01_016 R running task 0 17554 2 0x00000088 [ 1048.223414] Call Trace: [ 1048.223416] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.223419] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.223420] [] ? _raw_spin_lock+0x30/0x40 [ 1048.223427] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.223435] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.223437] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.223445] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.223469] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.223493] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.223495] [] ? check_preempt_curr+0x80/0xa0 [ 1048.223520] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.223544] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.223573] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.223602] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.223603] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.223629] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.223655] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.223656] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.223658] [] ? sched_feat_set+0xf0/0xf0 [ 1048.223683] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.223708] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.223711] [] ? kthread+0xd1/0xe0 [ 1048.223712] [] ? insert_kthread_work+0x40/0x40 [ 1048.223714] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.223716] [] ? insert_kthread_work+0x40/0x40 [ 1048.223717] Task dump for CPU 15: [ 1048.223719] mdt01_007 R running task 0 17310 2 0x00000088 [ 1048.223719] Call Trace: [ 1048.223721] [] ? native_queued_spin_lock_slowpath+0x122/0x200 [ 1048.223723] [] ? queued_spin_lock_slowpath+0xb/0xf [ 1048.223725] [] ? _raw_spin_lock+0x30/0x40 [ 1048.223731] [] ? cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1048.223739] [] ? lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1048.223741] [] ? wake_up_atomic_t+0x30/0x30 [ 1048.223749] [] ? LNetPrimaryNID+0xd5/0x220 [lnet] [ 1048.223773] [] ? ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1048.223797] [] ? ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1048.223799] [] ? check_preempt_curr+0x80/0xa0 [ 1048.223822] [] ? ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1048.223846] [] ? ptlrpc_error+0x10/0x20 [ptlrpc] [ 1048.223876] [] ? tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1048.223904] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1048.223906] [] ? __getnstimeofday64+0x3f/0xd0 [ 1048.223932] [] ? ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1048.223957] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1048.223958] [] ? __wake_up_common_lock+0x91/0xc0 [ 1048.223960] [] ? sched_feat_set+0xf0/0xf0 [ 1048.223985] [] ? ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1048.224009] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1048.224011] [] ? kthread+0xd1/0xe0 [ 1048.224013] [] ? insert_kthread_work+0x40/0x40 [ 1048.224015] [] ? ret_from_fork_nospec_begin+0x21/0x21 [ 1048.224017] [] ? insert_kthread_work+0x40/0x40 [ 1049.075830] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [mdt00_008:17320] [ 1049.075857] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1049.075878] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1049.075880] CPU: 0 PID: 17320 Comm: mdt00_008 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1049.075881] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1049.075882] task: ffff98321989e300 ti: ffff98321cd1c000 task.ti: ffff98321cd1c000 [ 1049.075884] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x122/0x200 [ 1049.075885] RSP: 0018:ffff98321cd1fab0 EFLAGS: 00000246 [ 1049.075886] RAX: 0000000000000000 RBX: ffff97f2bfa00000 RCX: 0000000000010000 [ 1049.075887] RDX: ffff9832bf4db8c0 RSI: 0000000000d90001 RDI: ffff97f2a83bcf40 [ 1049.075888] RBP: ffff98321cd1fab0 R08: ffff97f2bfa1b8c0 R09: 0000000000000000 [ 1049.075888] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98321cd1fa78 [ 1049.075889] R13: ffffffffa00ec9b1 R14: ffff98321cd1fa58 R15: ffff98321cd1fa88 [ 1049.075890] FS: 0000000000000000(0000) GS:ffff97f2bfa00000(0000) knlGS:0000000000000000 [ 1049.075891] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1049.075892] CR2: 00007ffff7ff8000 CR3: 0000007f0e586000 CR4: 00000000003607f0 [ 1049.075893] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1049.075894] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1049.075894] Call Trace: [ 1049.075896] [] queued_spin_lock_slowpath+0xb/0xf [ 1049.075898] [] _raw_spin_lock+0x30/0x40 [ 1049.075905] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1049.075913] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1049.075915] [] ? wake_up_atomic_t+0x30/0x30 [ 1049.075923] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1049.075948] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1049.075972] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1049.075974] [] ? check_preempt_curr+0x80/0xa0 [ 1049.075997] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1049.076020] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1049.076050] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1049.076078] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1049.076080] [] ? __getnstimeofday64+0x3f/0xd0 [ 1049.076106] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1049.076131] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1049.076132] [] ? __wake_up_common_lock+0x91/0xc0 [ 1049.076134] [] ? sched_feat_set+0xf0/0xf0 [ 1049.076158] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1049.076160] [] ? __switch_to+0xce/0x5a0 [ 1049.076184] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1049.076186] [] kthread+0xd1/0xe0 [ 1049.076188] [] ? insert_kthread_work+0x40/0x40 [ 1049.076189] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1049.076191] [] ? insert_kthread_work+0x40/0x40 [ 1049.076210] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [ 1049.089832] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [ll_mgs_0009:8655] [ 1049.089860] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1049.089881] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1049.089883] CPU: 3 PID: 8655 Comm: ll_mgs_0009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1049.089884] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1049.089885] task: ffff983229036300 ti: ffff98322904c000 task.ti: ffff98322904c000 [ 1049.089889] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1049.089889] RSP: 0018:ffff98322904fa50 EFLAGS: 00000246 [ 1049.089890] RAX: 0000000000000000 RBX: ffff97f2bfac0000 RCX: 0000000000190000 [ 1049.089891] RDX: ffff9832bf39b8c0 RSI: 0000000000710000 RDI: ffff97f2a83bcf40 [ 1049.089892] RBP: ffff98322904fa50 R08: ffff97f2bfadb8c0 R09: 0000000000000000 [ 1049.089893] R10: 0000000000000001 R11: 0000000000000001 R12: ffff98322904fa18 [ 1049.089894] R13: ffffffffa00ec9b1 R14: ffff98322904f9f8 R15: ffff98322904fa28 [ 1049.089895] FS: 0000000000000000(0000) GS:ffff97f2bfac0000(0000) knlGS:0000000000000000 [ 1049.089896] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1049.089897] CR2: 00007ffff579f624 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 1049.089898] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1049.089899] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1049.089899] Call Trace: [ 1049.089902] [] queued_spin_lock_slowpath+0xb/0xf [ 1049.089904] [] _raw_spin_lock+0x30/0x40 [ 1049.089911] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1049.089919] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1049.089921] [] ? wake_up_atomic_t+0x30/0x30 [ 1049.089929] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1049.089953] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1049.089977] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1049.089985] [] ? lnet_ptl_attach_md+0x38e/0x540 [lnet] [ 1049.089987] [] ? enqueue_task_fair+0x208/0x6c0 [ 1049.089989] [] ? sched_clock_cpu+0xa5/0xe0 [ 1049.089990] [] ? check_preempt_curr+0x80/0xa0 [ 1049.089992] [] ? ttwu_do_wakeup+0x19/0x100 [ 1049.090022] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1049.090050] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1049.090052] [] ? __getnstimeofday64+0x3f/0xd0 [ 1049.090078] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1049.090103] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1049.090104] [] ? __wake_up_common_lock+0x91/0xc0 [ 1049.090106] [] ? sched_feat_set+0xf0/0xf0 [ 1049.090130] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1049.090132] [] ? __switch_to+0xce/0x5a0 [ 1049.090156] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1049.090158] [] kthread+0xd1/0xe0 [ 1049.090160] [] ? insert_kthread_work+0x40/0x40 [ 1049.090162] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1049.090163] [] ? insert_kthread_work+0x40/0x40 [ 1049.090183] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1049.097833] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [mdt00_002:10663] [ 1049.097861] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1049.097882] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1049.097884] CPU: 5 PID: 10663 Comm: mdt00_002 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1049.097885] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1049.097886] task: ffff97f28e72a100 ti: ffff97f28e600000 task.ti: ffff97f28e600000 [ 1049.097889] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1049.097890] RSP: 0018:ffff97f28e603ab0 EFLAGS: 00000246 [ 1049.097891] RAX: 0000000000000000 RBX: ffff97f2bfb40000 RCX: 0000000000290000 [ 1049.097892] RDX: ffff9832bf55b8c0 RSI: 0000000000e90001 RDI: ffff97f2a83bcf40 [ 1049.097893] RBP: ffff97f28e603ab0 R08: ffff97f2bfb5b8c0 R09: 0000000000000000 [ 1049.097893] R10: 0000000000000001 R11: 0000000000000001 R12: ffff97f28e603a78 [ 1049.097894] R13: ffffffffa00ec9b1 R14: ffff97f28e603a58 R15: ffff97f28e603a88 [ 1049.097895] FS: 0000000000000000(0000) GS:ffff97f2bfb40000(0000) knlGS:0000000000000000 [ 1049.097896] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1049.097897] CR2: 00007ffff1096160 CR3: 0000007f0e586000 CR4: 00000000003607e0 [ 1049.097898] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1049.097899] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1049.097899] Call Trace: [ 1049.097902] [] queued_spin_lock_slowpath+0xb/0xf [ 1049.097904] [] _raw_spin_lock+0x30/0x40 [ 1049.097911] [] cfs_percpt_lock+0x58/0x110 [libcfs] [ 1049.097916] [] ? cfs_percpt_unlock+0x1a/0xe0 [libcfs] [ 1049.097925] [] lnet_discover_peer_locked+0x252/0x450 [lnet] [ 1049.097927] [] ? wake_up_atomic_t+0x30/0x30 [ 1049.097935] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1049.097959] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1049.097983] [] ptlrpc_send_reply+0x394/0x840 [ptlrpc] [ 1049.098009] [] ? lustre_pack_reply_flags+0x6f/0x1e0 [ptlrpc] [ 1049.098035] [] ptlrpc_send_error+0x9b/0x1b0 [ptlrpc] [ 1049.098058] [] ptlrpc_error+0x10/0x20 [ptlrpc] [ 1049.098088] [] tgt_request_handle+0x988/0x1570 [ptlrpc] [ 1049.098116] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1049.098117] [] ? __getnstimeofday64+0x3f/0xd0 [ 1049.098143] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1049.098167] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1049.098169] [] ? __wake_up_common_lock+0x91/0xc0 [ 1049.098170] [] ? sched_feat_set+0xf0/0xf0 [ 1049.098195] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1049.098197] [] ? __switch_to+0xce/0x5a0 [ 1049.098221] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1049.098223] [] kthread+0xd1/0xe0 [ 1049.098225] [] ? insert_kthread_work+0x40/0x40 [ 1049.098227] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1049.098228] [] ? insert_kthread_work+0x40/0x40 [ 1049.098248] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1057.232770] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [mdt00_009:17321] [ 1057.232799] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1057.232820] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1057.232822] CPU: 22 PID: 17321 Comm: mdt00_009 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1057.232823] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1057.232824] task: ffff983219899080 ti: ffff98321ca50000 task.ti: ffff98321ca50000 [ 1057.232827] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1057.232828] RSP: 0018:ffff98321ca53a50 EFLAGS: 00000246 [ 1057.232829] RAX: 0000000000000000 RBX: ffff97f2bfd80000 RCX: 0000000000b10000 [ 1057.232830] RDX: ffff9832bf3db8c0 RSI: 0000000000790000 RDI: ffff97f2a83bcf40 [ 1057.232831] RBP: ffff98321ca53a50 R08: ffff97f2bfd9b8c0 R09: 0000000000000000 [ 1057.232832] R10: 0000000000000002 R11: ffff983206cd2600 R12: ffff98321ca53a18 [ 1057.232833] R13: ffffffffa00ec9b1 R14: ffff98321ca539f8 R15: ffff98321ca53a28 [ 1057.232834] FS: 0000000000000000(0000) GS:ffff97f2bfd80000(0000) knlGS:0000000000000000 [ 1057.232835] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1057.232835] CR2: 00007ffff1096160 CR3: 0000002e8de10000 CR4: 00000000003607e0 [ 1057.232836] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1057.232837] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1057.232838] Call Trace: [ 1057.232840] [] queued_spin_lock_slowpath+0xb/0xf [ 1057.232842] [] _raw_spin_lock+0x30/0x40 [ 1057.232849] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [ 1057.232857] [] lnet_discover_peer_locked+0xa0/0x450 [lnet] [ 1057.232859] [] ? wake_up_atomic_t+0x30/0x30 [ 1057.232867] [] LNetPrimaryNID+0xd5/0x220 [lnet] [ 1057.232892] [] ptlrpc_connection_get+0x3e/0x450 [ptlrpc] [ 1057.232916] [] target_handle_connect+0x12f1/0x2b90 [ptlrpc] [ 1057.232940] [] ? ptlrpc_send_reply+0x2ae/0x840 [ptlrpc] [ 1057.232942] [] ? enqueue_task_fair+0x208/0x6c0 [ 1057.232944] [] ? sched_clock_cpu+0xa5/0xe0 [ 1057.232945] [] ? check_preempt_curr+0x80/0xa0 [ 1057.232947] [] ? ttwu_do_wakeup+0x19/0x100 [ 1057.232977] [] tgt_request_handle+0x4fa/0x1570 [ptlrpc] [ 1057.233006] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [ 1057.233007] [] ? __getnstimeofday64+0x3f/0xd0 [ 1057.233033] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [ 1057.233057] [] ? ptlrpc_wait_event+0xb8/0x370 [ptlrpc] [ 1057.233059] [] ? __wake_up_common_lock+0x91/0xc0 [ 1057.233060] [] ? sched_feat_set+0xf0/0xf0 [ 1057.233084] [] ptlrpc_main+0xc49/0x1c50 [ptlrpc] [ 1057.233086] [] ? __switch_to+0xce/0x5a0 [ 1057.233110] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [ 1057.233112] [] kthread+0xd1/0xe0 [ 1057.233114] [] ? insert_kthread_work+0x40/0x40 [ 1057.233116] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1057.233117] [] ? insert_kthread_work+0x40/0x40 [ 1057.233137] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1060.186625] NMI watchdog: Watchdog detected hard LOCKUP on cpu 9 [ 1060.186665] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgs(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) ko2iblnd(OE) lnet(OE) libcfs(OE) iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi rpcrdma ib_iser kvm irqbypass joydev pcspkr lpc_ich i2c_i801 ioatdma ses enclosure sg ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter dm_round_robin acpi_cpufreq ib_ipoib rdma_ucm ib_umad sch_fq_codel iw_cxgb4 rdma_cm iw_cm ib_cm iw_cxgb3 zfs(POE) binfmt_misc zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) msr_safe(OE) ip_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache overlay(T) ext4 mbcache jbd2 dm_service_time sd_mod crc_t10dif crct10dif_generic mlx5_ib ib_uverbs [ 1060.186696] ib_core be2iscsi bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio libcxgbi libcxgb qla4xxx iscsi_boot_sysfs mgag200 i2c_algo_bit 8021q drm_kms_helper garp syscopyarea mrp crct10dif_pclmul stp sysfillrect crct10dif_common sysimgblt crc32_pclmul llc fb_sys_fops crc32c_intel ttm ghash_clmulni_intel mlx5_core drm mxm_wmi ahci ixgbe(OE) aesni_intel mpt3sas mlxfw dm_multipath lrw devlink libahci gf128mul dca glue_helper ablk_helper raid_class ptp drm_panel_orientation_quirks libata cryptd scsi_transport_sas pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse [ 1060.186698] CPU: 9 PID: 9 Comm: rcu_sched Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1060.186699] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1060.186700] task: ffff97b4ded39080 ti: ffff97b4ded40000 task.ti: ffff97b4ded40000 [ 1060.186704] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x126/0x200 [ 1060.186705] RSP: 0018:ffff97b4ded43d88 EFLAGS: 00000046 [ 1060.186706] RAX: 0000000000000000 RBX: 0000000000000246 RCX: 0000000000490000 [ 1060.186707] RDX: ffff9832bf55b8d0 RSI: 0000000000eb0101 RDI: ffffffffa0c7a5c0 [ 1060.186708] RBP: ffff97b4ded43d88 R08: ffff9832bf25b8c0 R09: 0000000000000000 [ 1060.186708] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000010000 [ 1060.186709] R13: ffffffffa0c7a3c0 R14: 0000000000000000 R15: ffffffffa0c7a5c0 [ 1060.186710] FS: 0000000000000000(0000) GS:ffff9832bf240000(0000) knlGS:0000000000000000 [ 1060.186711] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1060.186712] CR2: 00007ffff7ff8000 CR3: 0000007ecf0f6000 CR4: 00000000003607e0 [ 1060.186713] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1060.186714] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1060.186715] Call Trace: [ 1060.186717] [] queued_spin_lock_slowpath+0xb/0xf [ 1060.186719] [] _raw_spin_lock_irqsave+0x47/0x50 [ 1060.186721] [] force_qs_rnp+0x67/0x180 [ 1060.186723] [] ? rcu_eqs_exit+0xa0/0xa0 [ 1060.186725] [] rcu_gp_fqs+0x39/0xa0 [ 1060.186727] [] rcu_gp_kthread+0x430/0x7f0 [ 1060.186729] [] ? wake_up_atomic_t+0x30/0x30 [ 1060.186731] [] ? rcu_gp_fqs+0xa0/0xa0 [ 1060.186733] [] kthread+0xd1/0xe0 [ 1060.186735] [] ? insert_kthread_work+0x40/0x40 [ 1060.186737] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1060.186738] [] ? insert_kthread_work+0x40/0x40 [ 1060.186758] Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 17 d5 a0 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [ 1060.186759] Kernel panic - not syncing: Hard LOCKUP [ 1060.186761] CPU: 9 PID: 9 Comm: rcu_sched Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.53.1.1chaos.ch6.x86_64 #1 [ 1060.186762] Hardware name: Intel Corporation S2600WTTR/S2600WTTR, BIOS SE5C610.86B.01.01.0024.021320181901 02/13/2018 [ 1060.186762] Call Trace: [ 1060.186765] [] dump_stack+0x19/0x1b [ 1060.186767] [] panic+0xe8/0x21f [ 1060.186770] [] ? show_regs+0x58/0x290 [ 1060.186774] [] nmi_panic+0x3f/0x40 [ 1060.186779] [] watchdog_overflow_callback+0x119/0x140 [ 1060.186783] [] __perf_event_overflow+0x57/0x100 [ 1060.186786] [] perf_event_overflow+0x14/0x20 [ 1060.186789] [] handle_pmi_common+0x1a0/0x250 [ 1060.186794] [] ? ioremap_page_range+0x2e8/0x480 [ 1060.186798] [] ? vunmap_page_range+0x234/0x470 [ 1060.186800] [] ? unmap_kernel_range_noflush+0x11/0x20 [ 1060.186806] [] ? ghes_copy_tofrom_phys+0x120/0x230 [ 1060.186808] [] intel_pmu_handle_irq+0xcf/0x1d0 [ 1060.186810] [] perf_event_nmi_handler+0x31/0x50 [ 1060.186812] [] nmi_handle.isra.0+0x9c/0x170 [ 1060.186814] [] do_nmi+0x165/0x450 [ 1060.186816] [] end_repeat_nmi+0x1e/0x81 [ 1060.186818] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1060.186820] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1060.186821] [] ? native_queued_spin_lock_slowpath+0x126/0x200 [ 1060.186824] [] queued_spin_lock_slowpath+0xb/0xf [ 1060.186825] [] _raw_spin_lock_irqsave+0x47/0x50 [ 1060.186827] [] force_qs_rnp+0x67/0x180 [ 1060.186829] [] ? rcu_eqs_exit+0xa0/0xa0 [ 1060.186831] [] rcu_gp_fqs+0x39/0xa0 [ 1060.186832] [] rcu_gp_kthread+0x430/0x7f0 [ 1060.186834] [] ? wake_up_atomic_t+0x30/0x30 [ 1060.186836] [] ? rcu_gp_fqs+0xa0/0xa0 [ 1060.186838] [] kthread+0xd1/0xe0 [ 1060.186839] [] ? insert_kthread_work+0x40/0x40 [ 1060.186841] [] ret_from_fork_nospec_begin+0x21/0x21 [ 1060.186843] [] ? insert_kthread_work+0x40/0x40