-
Bug
-
Resolution: Unresolved
-
Medium
-
None
-
None
-
None
-
3
-
9223372036854775807
Client on soak setup crashed when IOs were in progress and we brought down the switch ports for one of the exa node that was hosting some osts and mdts
Client node timing
[root@co-es-pm-218 ~]# date Fri Sep 19 11:00:13 AM UTC 2025 [root@co-es-pm-218 ~]# uptime 11:00:14 up 4 min, 1 user, load average: 0.01, 0.06, 0.02 On client node from vmcore-dmesg.txt: [252396.877058] LNet: 2 peer NIs in recovery (showing 2): 172.16.13.2@tcp, 172.16.13.1@tcp [252396.885524] Lustre: 122992:0:(llite_lib.c:3787:ll_dirty_page_discard_warn()) testfs: dirty page discard: 172.16.13.3@tcp,172.16.13.4@tcp:172.16.13.7@tcp,172.16.13.8@tcp:172.16.13.5@tcp,172.16.13.6@tcp:172.16.13.1@tcp,172.16.13.2@tcp:/testfs/fid: [0x2c0002be1:0xc0e4:0x0]// may get corrupted (rc -108) [252456.887626] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [252458.842200] Lustre: testfs-OST0000-osc-ffff92986bc99800: Connection restored to 172.16.13.3@tcp (at 172.16.13.3@tcp) [252458.842242] rcu: 44-...!: (1 GPs behind) idle=4c94/1/0x4000000000000002 softirq=21094089/21094089 fqs=389 [252475.615569] NMI watchdog: Watchdog detected hard LOCKUP on cpu 17 [252475.615572] Modules linked in: mgc(OE) lustre(OE) mdc(OE) fid(OE) lov(OE) osc(OE) lmv(OE) fld(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfio_pci vfio_pci_core vfio_iommu_type1 vfio iommufd rfkill rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) sunrpc amd_atl ipmi_ssif intel_rapl_msr intel_rapl_common amd64_edac edac_mce_amd kvm_amd kvm ast drm_shmem_helper irqbypass drm_kms_helper rapl wmi_bmof acpi_cpufreq pcspkr acpi_ipmi syscopyarea sysfillrect ipmi_si sysimgblt fb_sys_fops ptdma i2c_piix4 k10temp ipmi_devintf ipmi_msghandler fuse drm ext4 mbcache jbd2 mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) sd_mod t10_pi sg mlx5_core(OE) mlxfw(OE) crct10dif_pclmul macsec crc32_pclmul crc32c_intel tls ahci libahci ghash_clmulni_intel psample mlxdevm(OE) igb mlx_compat(OE) i2c_algo_bit libata ccp pci_hyperv_intf sp5100_tco dca wmi xpmem(OE) [last unloaded: libcfs] [252475.615630] CPU: 17 PID: 16 Comm: rcu_preempt Kdump: loaded Tainted: G OE ------- --- 5.14.0-427.42.1.el9_4.x86_64 #1 [252475.615633] Hardware name: Bull SAS H252-Z10-00/MZ12-HD0-00, BIOS M14a 03/10/2023 [252475.615634] RIP: 0010:native_queued_spin_lock_slowpath+0x76/0x2b0 [252475.615640] Code: 8b 45 00 0f b6 d2 c1 e2 08 30 e4 09 d0 a9 00 01 ff ff 0f 85 db 01 00 00 85 c0 74 12 0f b6 45 00 84 c0 74 0a f3 90 0f b6 45 00 <84> c0 75 f6 b8 01 00 00 00 66 89 45 00 5b 5d 41 5c 41 5d e9 fd c1 [252475.615642] RSP: 0018:ffffab50801a7e18 EFLAGS: 00000002 [252475.615643] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000002 [252475.615645] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff8b1ea580 [252475.615646] RBP: ffffffff8b1ea580 R08: ffffffff8b23b4e0 R09: ffff929b2e2739b8 [252475.615646] R10: 00000000000003c8 R11: 00000000000003c5 R12: 0000000000000246 [252475.615647] R13: 0000000000000000 R14: 0000000000033ec0 R15: ffffffff8b1ea580 [252475.615648] FS: 0000000000000000(0000) GS:ffff929b2e240000(0000) knlGS:0000000000000000 [252475.615650] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [252475.615651] CR2: 0000150e10001048 CR3: 000000010f1ca002 CR4: 0000000000770ee0 [252475.615652] PKRU: 55555554 [252475.615653] Call Trace: [252475.615654] <NMI> [252475.615655] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615658] ? show_trace_log_lvl+0x26e/0x2df [252475.615663] ? show_trace_log_lvl+0x26e/0x2df [252475.615667] ? _raw_spin_lock_irqsave+0x30/0x40 [252475.615671] ? watchdog_overflow_callback+0xd0/0x130 [252475.615674] ? __perf_event_overflow+0x105/0x1e0 [252475.615677] ? x86_pmu_handle_irq+0x147/0x1b0 [252475.615684] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615686] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615688] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615690] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615691] ? ghes_copy_tofrom_phys+0x74/0x120 [252475.615697] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615699] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615701] ? ghes_in_nmi_queue_one_entry.constprop.0+0x3b/0x2a0 [252475.615704] ? amd_pmu_handle_irq+0x42/0xb0 [252475.615706] ? perf_event_nmi_handler+0x28/0x50 [252475.615708] ? nmi_handle+0x5e/0x120 [252475.615711] ? default_do_nmi+0x40/0x130 [252475.615715] ? exc_nmi+0xff/0x130 [252475.615717] ? end_repeat_nmi+0x16/0x67 [252475.615721] ? native_queued_spin_lock_slowpath+0x76/0x2b0 [252475.615723] ? native_queued_spin_lock_slowpath+0x76/0x2b0 [252475.615727] </NMI> [252475.615727] <TASK> [252475.615729] _raw_spin_lock_irqsave+0x30/0x40 [252475.615731] force_qs_rnp+0xf0/0x250 [252475.615734] ? __pfx_rcu_implicit_dynticks_qs+0x10/0x10 [252475.615736] ? __pfx_rcu_gp_kthread+0x10/0x10 [252475.615738] rcu_gp_fqs_loop+0x346/0x460 [252475.615741] rcu_gp_kthread+0xfb/0x190 [252475.615744] kthread+0xe0/0x100 [252475.615747] ? __pfx_kthread+0x10/0x10 [252475.615749] ret_from_fork+0x2c/0x50 [252475.615755] </TASK> [252475.615756] Kernel panic - not syncing: Hard LOCKUP [252475.615757] CPU: 17 PID: 16 Comm: rcu_preempt Kdump: loaded Tainted: G OE ------- --- 5.14.0-427.42.1.el9_4.x86_64 #1 [252475.615759] Hardware name: Bull SAS H252-Z10-00/MZ12-HD0-00, BIOS M14a 03/10/2023 [252475.615759] Call Trace: [252475.615761] <NMI> [252475.615762] dump_stack_lvl+0x34/0x48 [252475.615765] panic+0x107/0x2f7 [252475.615770] nmi_panic.cold+0xc/0xc [252475.615772] watchdog_overflow_callback+0x11f/0x130 [252475.615774] __perf_event_overflow+0x105/0x1e0 [252475.615776] x86_pmu_handle_irq+0x147/0x1b0 [252475.615782] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615784] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615786] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615788] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615790] ? ghes_copy_tofrom_phys+0x74/0x120 [252475.615793] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615795] ? srso_alias_return_thunk+0x5/0xfbef5 [252475.615797] ? ghes_in_nmi_queue_one_entry.constprop.0+0x3b/0x2a0 [252475.615799] amd_pmu_handle_irq+0x42/0xb0 [252475.615802] perf_event_nmi_handler+0x28/0x50 [252475.615804] nmi_handle+0x5e/0x120 [252475.615806] default_do_nmi+0x40/0x130 [252475.615809] exc_nmi+0xff/0x130 [252475.615811] end_repeat_nmi+0x16/0x67 [252475.615813] RIP: 0010:native_queued_spin_lock_slowpath+0x76/0x2b0 [252475.615815] Code: 8b 45 00 0f b6 d2 c1 e2 08 30 e4 09 d0 a9 00 01 ff ff 0f 85 db 01 00 00 85 c0 74 12 0f b6 45 00 84 c0 74 0a f3 90 0f b6 45 00 <84> c0 75 f6 b8 01 00 00 00 66 89 45 00 5b 5d 41 5c 41 5d e9 fd c1 [252475.615816] RSP: 0018:ffffab50801a7e18 EFLAGS: 00000002 [252475.615817] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000002 [252475.615818] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff8b1ea580 [252475.615819] RBP: ffffffff8b1ea580 R08: ffffffff8b23b4e0 R09: ffff929b2e2739b8 [252475.615820] R10: 00000000000003c8 R11: 00000000000003c5 R12: 0000000000000246 [252475.615821] R13: 0000000000000000 R14: 0000000000033ec0 R15: ffffffff8b1ea580 [252475.615825] ? native_queued_spin_lock_slowpath+0x76/0x2b0 [252475.615827] ? native_queued_spin_lock_slowpath+0x76/0x2b0 [252475.615829] </NMI> [252475.615829] <TASK> [252475.615830] _raw_spin_lock_irqsave+0x30/0x40 [252475.615832] force_qs_rnp+0xf0/0x250 [252475.615834] ? __pfx_rcu_implicit_dynticks_qs+0x10/0x10 [252475.615836] ? __pfx_rcu_gp_kthread+0x10/0x10 [252475.615838] rcu_gp_fqs_loop+0x346/0x460 [252475.615841] rcu_gp_kthread+0xfb/0x190 [252475.615843] kthread+0xe0/0x100 [252475.615845] ? __pfx_kthread+0x10/0x10 [252475.615847] ret_from_fork+0x2c/0x50 [252475.615852] </TASK>
Server
[root@ai400x2-c0-vm01 ~]# emf ssh run 'journalctl -k | grep -A20 "Call Trace" ' ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: Call Trace TBD: ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ptlrpc_set_wait+0x4ba/0x6d0 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ptlrpc_queue_wait+0x81/0x220 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] osp_remote_sync+0x155/0x1d0 [osp] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] osp_xattr_get+0x89c/0x1150 [osp] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] lod_xattr_get+0xdc/0x820 [lod] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] mdd_xattr_get+0x82/0x4a0 [mdd] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] mdt_pack_secctx_in_reply+0xe3/0x2f0 [mdt] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] mdt_getattr_name_lock+0x960/0x1e90 [mdt] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] mdt_intent_getattr+0x264/0x440 [mdt] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] mdt_intent_opc+0x452/0xa80 [mdt] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] mdt_intent_policy+0x1fd/0x390 [mdt] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ldlm_lock_enqueue+0x3a8/0x800 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ldlm_handle_enqueue0+0x79d/0x16c0 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] tgt_enqueue+0xa4/0x200 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] tgt_request_handle+0xc0a/0x1b00 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ptlrpc_server_handle_request+0x25a/0xca0 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ptlrpc_main+0xb45/0x13a0 [ptlrpc] ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] kthread+0x134/0x150 ai400x2-c0-vm01: Sep 19 10:49:40 ai400x2-c0-vm01 kernel: [<0>] ret_from_fork+0x1f/0x40 ai400x2-c0-vm01: Sep 19 10:49:53 ai400x2-c0-vm01 kernel: LDISKFS-fs (sdd): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: Call Trace TBD: ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] osp_precreate_reserve+0x5e9/0x9b0 [osp] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] osp_declare_create+0x48c/0x6a0 [osp] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_sub_declare_create+0x10b/0x260 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_qos_declare_object_on+0x1c1/0x400 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_ost_alloc_rr+0xaef/0x14d0 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_qos_prep_create+0xc66/0xee0 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_prepare_create+0x229/0x300 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_declare_striped_create+0x279/0x780 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] lod_declare_create+0x2c7/0x640 [lod] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdd_declare_create_object_internal+0xcd/0x360 [mdd] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdd_declare_create_object.isra.38+0x49/0x7e0 [mdd] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdd_declare_create+0x72/0x500 [mdd] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdd_create+0x7e4/0x1860 [mdd] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_open+0x27b5/0x3320 [mdt] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_rec+0x127/0x260 [mdt] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_internal+0x4ac/0x7a0 [mdt] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdt_intent_open+0x128/0x3c0 [mdt] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdt_intent_opc+0x452/0xa80 [mdt] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] mdt_intent_policy+0x1fd/0x390 [mdt] ai400x2-c1-vm00: Sep 18 17:11:25 ai400x2-c1-vm00 kernel: [<0>] ldlm_lock_enqueue+0x3a8/0x800 [ptlrpc] ai400x2-c1-vm00: – ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: Call Trace TBD: ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] __jbd2_log_wait_for_space+0xc5/0x1a0 [jbd2] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] add_transaction_credits+0x281/0x290 [jbd2] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] start_this_handle+0x10a/0x520 [jbd2] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] jbd2__journal_start+0xee/0x1f0 [jbd2] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] __ldiskfs_journal_start_sb+0x6e/0x140 [ldiskfs] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] osd_trans_start+0x289/0x520 [osd_ldiskfs] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ofd_destroy+0x2ef/0xad0 [ofd] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ofd_destroy_by_fid+0x39e/0x540 [ofd] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ofd_orphans_destroy+0x354/0x8f0 [ofd] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ofd_create_hdl+0x17d2/0x18f0 [ofd] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] tgt_request_handle+0xc0a/0x1b00 [ptlrpc] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ptlrpc_server_handle_request+0x25a/0xca0 [ptlrpc] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ptlrpc_main+0xb45/0x13a0 [ptlrpc] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] kthread+0x134/0x150 ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] ret_from_fork+0x1f/0x40 ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: Pid: 57700, comm: mdt02_017 4.18.0-553.71.1.el8_lustre.ddn17.x86_64 #1 SMP Fri Sep 12 00:41:32 UTC 2025 ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: Call Trace TBD: ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] osp_precreate_reserve+0x5e9/0x9b0 [osp] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] osp_declare_create+0x48c/0x6a0 [osp] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] lod_sub_declare_create+0x10b/0x260 [lod] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] lod_qos_declare_object_on+0x1c1/0x400 [lod] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] lod_ost_alloc_rr+0xaef/0x14d0 [lod] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] lod_qos_prep_create+0xc66/0xee0 [lod] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] lod_prepare_create+0x229/0x300 [lod] ai400x2-c1-vm00: Sep 18 17:11:29 ai400x2-c1-vm00 kernel: [<0>] lod_declare_striped_create+0x279/0x780 [lod] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] lod_declare_create+0x2c7/0x640 [lod] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdd_declare_create_object_internal+0xcd/0x360 [mdd] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdd_declare_create_object.isra.38+0x49/0x7e0 [mdd] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdd_declare_create+0x72/0x500 [mdd] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdd_create+0x7e4/0x1860 [mdd] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_open+0x27b5/0x3320 [mdt] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_rec+0x127/0x260 [mdt] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_internal+0x4ac/0x7a0 [mdt] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdt_intent_open+0x128/0x3c0 [mdt] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdt_intent_opc+0x452/0xa80 [mdt] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] mdt_intent_policy+0x1fd/0x390 [mdt] ai400x2-c1-vm00: Sep 18 17:11:30 ai400x2-c1-vm00 kernel: [<0>] ldlm_lock_enqueue+0x3a8/0x800 [ptlrpc] ai400x2-c1-vm00: – ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: Call Trace TBD: ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] ldlm_completion_ast+0x8e5/0xb30 [ptlrpc] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] ldlm_cli_enqueue_fini+0xa12/0xee0 [ptlrpc] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] ldlm_cli_enqueue+0x5be/0x9d0 [ptlrpc] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] osp_md_object_lock+0x1a6/0x2b0 [osp] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] lod_object_lock+0x134/0x750 [lod] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_remote_object_lock_try+0x186/0x3c0 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_object_lock_internal+0xa4/0x430 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_object_lock+0x1b/0x30 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_create+0x70a/0xe60 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_create+0x317/0x3d0 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_rec+0x127/0x260 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_reint_internal+0x4ac/0x7a0 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] mdt_reint+0x5e/0x100 [mdt] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] tgt_request_handle+0xc0a/0x1b00 [ptlrpc] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] ptlrpc_server_handle_request+0x25a/0xca0 [ptlrpc] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] ptlrpc_main+0xb45/0x13a0 [ptlrpc] ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] kthread+0x134/0x150 ai400x2-c1-vm00: Sep 19 10:49:42 ai400x2-c1-vm00 kernel: [<0>] ret_from_fork+0x1f/0x40 ai400x2-c1-vm00: Sep 19 10:49:55 ai400x2-c1-vm00 kernel: Lustre: testfs-MDT0001: Received MDS connection from 172.16.13.3@tcp, removing former export from 172.16.13.1@tcp ai400x2-c1-vm00: Sep 19 10:49:55 ai400x2-c1-vm00 kernel: Lustre: mdt05_010: service thread pid 2003847 completed after 215.309s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).