crash> sys KERNEL: /usr/lib/debug/lib/modules/2.6.32-504.16.2.el6.Bull.74.x86_64/vmlinux DUMPFILE: vmcore [PARTIAL DUMP] CPUS: 32 DATE: Sun Jun 7 06:41:03 2015 UPTIME: 4 days, 19:14:07 LOAD AVERAGE: 646.72, 556.57, 328.17 TASKS: 2556 NODENAME: mds111 RELEASE: 2.6.32-504.16.2.el6.Bull.74.x86_64 VERSION: #1 SMP Tue Apr 28 01:43:42 CEST 2015 MACHINE: x86_64 (2266 Mhz) MEMORY: 64 GB PANIC: "Kernel panic - not syncing: LBUG" dmesg: --- <4>Lustre: DEBUG MARKER: Sun Jun 7 06:30:01 2015 <4> <4>Lustre: DEBUG MARKER: Sun Jun 7 06:35:02 2015 <4> <4>Lustre: DEBUG MARKER: Sun Jun 7 06:40:02 2015 <4> <4>Lustre: 16024:0:(service.c:1357:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply <4> req@ffff8802dc9b6c00 x1502885976724488/t0(0) o103->dfd416d2-63e3-7fc4-3da6-d92933f57357@X.Y.Z.43@o2ib11:0/0 lens 328/0 e 20 to 0 dl 1433652062 ref 2 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 16024:0:(service.c:1357:ptlrpc_at_send_early_reply()) Skipped 74325 previous similar messages <3>LustreError: 16046:0:(service.c:2020:ptlrpc_server_handle_request()) @@@ Dropping timed-out request from 12345-X.Y.Z.182@o2ib11: deadline 600:1s ago <3> req@ffff880459a46400 x1502885940363796/t0(0) o103->556243b7-70d3-14af-aa23-700d75524d16@X.Y.Z.182@o2ib11:0/0 lens 328/0 e 20 to 0 dl 1433652061 ref 1 fl Interpret:/0/ffffffff rc 0/-1 <4>Lustre: 16037:0:(service.c:2052:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (600:1s); client may timeout. req@ffff880459a46800 x1502884986437776/t0(0) o103->df950e50-7e01-86ac-a288-2c6416035e49@X.Y.Z.204@o2ib11:0/0 lens 328/0 e 20 to 0 dl 1433652061 ref 1 fl Interpret:/0/ffffffff rc 0/-1 <4>Lustre: 16037:0:(service.c:2052:ptlrpc_server_handle_request()) Skipped 22 previous similar messages <3>LustreError: 16046:0:(service.c:2020:ptlrpc_server_handle_request()) Skipped 15 previous similar messages <4>Lustre: ptmp2-MDT0000: Client 97f263bc-c460-54d0-e06b-3388849c01b7 (at X.Y.Z.156@o2ib11) reconnecting <3>LustreError: 12255:0:(sec.c:379:import_sec_validate_get()) import ffff880903ac1000 (FULL) with no sec <0>LustreError: 19842:0:(connection.c:104:ptlrpc_connection_put()) ASSERTION( atomic_read(&conn->c_refcount) > 1 ) failed: <0>LustreError: 19842:0:(connection.c:104:ptlrpc_connection_put()) LBUG <4>Pid: 19842, comm: obd_zombid <4> <4>Call Trace: <4> [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] <4> [] lbug_with_loc+0x47/0xb0 [libcfs] <4> [] ptlrpc_connection_put+0x1db/0x1e0 [ptlrpc] <4> [] class_import_destroy+0x5d/0x420 [obdclass] <4> [] obd_zombie_impexp_cull+0xcb/0x5d0 [obdclass] <4> [] obd_zombie_impexp_thread+0x65/0x190 [obdclass] <4> [] ? default_wake_function+0x0/0x20 <4> [] ? obd_zombie_impexp_thread+0x0/0x190 [obdclass] <4> [] kthread+0x9e/0xc0 <4> [] child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4> <0>Kernel panic - not syncing: LBUG <4>Pid: 19842, comm: obd_zombid Not tainted 2.6.32-504.16.2.el6.Bull.74.x86_64 #1 <4>Call Trace: <4> [] ? panic+0xa7/0x16f <4> [] ? lbug_with_loc+0x9b/0xb0 [libcfs] <4> [] ? ptlrpc_connection_put+0x1db/0x1e0 [ptlrpc] <4> [] ? class_import_destroy+0x5d/0x420 [obdclass] <4> [] ? obd_zombie_impexp_cull+0xcb/0x5d0 [obdclass] <4> [] ? obd_zombie_impexp_thread+0x65/0x190 [obdclass] <4> [] ? default_wake_function+0x0/0x20 <4> [] ? obd_zombie_impexp_thread+0x0/0x190 [obdclass] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 --- crash> bt PID: 19842 TASK: ffff88086f452040 CPU: 6 COMMAND: "obd_zombid" #0 [ffff8805516b5c08] machine_kexec at ffffffff8103b71b #1 [ffff8805516b5c68] crash_kexec at ffffffff810c9942 #2 [ffff8805516b5d38] panic at ffffffff8152a2c4 #3 [ffff8805516b5db8] lbug_with_loc at ffffffffa04f2eeb [libcfs] #4 [ffff8805516b5dd8] ptlrpc_connection_put at ffffffffa082938b [ptlrpc] #5 [ffff8805516b5df8] class_import_destroy at ffffffffa066c70d [obdclass] #6 [ffff8805516b5e28] obd_zombie_impexp_cull at ffffffffa067067b [obdclass] #7 [ffff8805516b5e68] obd_zombie_impexp_thread at ffffffffa0670be5 [obdclass] #8 [ffff8805516b5ee8] kthread at ffffffff8109e71e #9 [ffff8805516b5f48] kernel_thread at ffffffff8100c20a --- 96 int ptlrpc_connection_put(struct ptlrpc_connection *conn) 97 { 98 int rc = 0; 99 ENTRY; 100 101 if (!conn) 102 RETURN(rc); 103 104 LASSERT(cfs_atomic_read(&conn->c_refcount) > 1); crash> dis -rl 0xffffffffa066c70d /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 945 0xffffffffa066c6b0 : push %rbp 0xffffffffa066c6b1 : mov %rsp,%rbp 0xffffffffa066c6b4 : push %r13 0xffffffffa066c6b6 : push %r12 0xffffffffa066c6b8 : push %rbx 0xffffffffa066c6b9 : sub $0x8,%rsp 0xffffffffa066c6bd : nopl 0x0(%rax,%rax,1) /usr/src/debug/lustre-2.5.3.90/libcfs/include/libcfs/libcfs_debug.h: 220 0xffffffffa066c6c2 : mov -0x1488c4(%rip),%eax # 0xffffffffa0523e04 /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 945 0xffffffffa066c6c8 : mov %rdi,%r12 /usr/src/debug/lustre-2.5.3.90/libcfs/include/libcfs/libcfs_debug.h: 219 0xffffffffa066c6cb : test $0x1,%al 0xffffffffa066c6cd : je 0xffffffffa066c6dc 0xffffffffa066c6cf : testb $0x20,-0x1488d6(%rip) # 0xffffffffa0523e00 0xffffffffa066c6d6 : jne 0xffffffffa066c9a0 0xffffffffa066c6dc : test %al,%al 0xffffffffa066c6de : jns 0xffffffffa066c6ed 0xffffffffa066c6e0 : testb $0x20,-0x1488e7(%rip) # 0xffffffffa0523e00 0xffffffffa066c6e7 : jne 0xffffffffa066c930 /gestconf/project/hpcdelivery/build/current_AE4/export/x86_64_rhel6.4/kernel/usr/src/kernels/2.6.32-504.16.2.el6.Bull.74.x86_64/arch/x86/include/asm/atomic_64.h: 23 0xffffffffa066c6ed : mov 0x40(%r12),%eax /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 951 0xffffffffa066c6f2 : test %eax,%eax 0xffffffffa066c6f4 : jne 0xffffffffa066ca31 /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 953 0xffffffffa066c6fa : mov 0x50(%r12),%rdi <=== HERE /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 955 0xffffffffa066c6ff : lea 0x278(%r12),%r13 /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 953 0xffffffffa066c707 : callq *0xc0ddb(%rip) # 0xffffffffa072d4e8 /gestconf/project/hpcdelivery/build/current_AE4/export/x86_64_rhel6.4/kernel/usr/src/kernels/2.6.32-504.16.2.el6.Bull.74.x86_64/include/linux/list.h: 183 0xffffffffa066c70d : mov 0x278(%r12),%rbx 943 /* Import management functions */ 944 void class_import_destroy(struct obd_import *imp) 945 { 946 ENTRY; 947 948 CDEBUG(D_IOCTL, "destroying import %p for %s\n", imp, 949 imp->imp_obd->obd_name); 950 951 LASSERT_ATOMIC_ZERO(&imp->imp_refcount); 952 953 ptlrpc_put_connection_superhack(imp->imp_connection); <=== HERE 954 955 while (!cfs_list_empty(&imp->imp_conn_list)) { 956 struct obd_import_conn *imp_conn; 957 958 imp_conn = cfs_list_entry(imp->imp_conn_list.next, 959 struct obd_import_conn, oic_item); 960 cfs_list_del_init(&imp_conn->oic_item); 961 ptlrpc_put_connection_superhack(imp_conn->oic_conn); 962 OBD_FREE(imp_conn, sizeof(*imp_conn)); 963 } 964 965 LASSERT(imp->imp_sec == NULL); 966 class_decref(imp->imp_obd, "import", imp); 967 OBD_FREE_RCU(imp, sizeof(*imp), &imp->imp_handle); 968 EXIT; 969 } crash> dis ptlrpc_connection_put 0xffffffffa08291b0 : push %rbp 0xffffffffa08291b1 : mov %rsp,%rbp 0xffffffffa08291b4 : push %r12 0xffffffffa08291b6 : push %rbx [...] crash> bt -f PID: 19842 TASK: ffff88086f452040 CPU: 6 COMMAND: "obd_zombid" #0 [ffff8805516b5c08] machine_kexec at ffffffff8103b71b ffff8805516b5c10: 00000000030a1000 ffff8800030a1000 ffff8805516b5c20: 00000000030a0000 0000000000000000 ffff8805516b5c30: 8800000000000000 ffff880903acffff ffff8805516b5c40: 0000000000000000 ffff8805516b5c70 ffff8805516b5c50: ffff880903ac1278 ffffffffa072d530 ffff8805516b5c60: ffff8805516b5d30 ffffffff810c9942 #1 [ffff8805516b5c68] crash_kexec at ffffffff810c9942 ffff8805516b5c70: ffffffffa072d530 ffffffffa072d530 ffff8805516b5c80: ffff880903ac1278 ffff8808bb14cf40 ffff8805516b5c90: ffff8805516b5d30 0000000000000000 ffff8805516b5ca0: 0000000000000000 0000000000000001 ffff8805516b5cb0: ffffffff816490a0 0000000000000000 ffff8805516b5cc0: 0000000000000001 0000000000002ab4 ffff8805516b5cd0: 0000000000000000 0000000000000001 ffff8805516b5ce0: 0000000000000002 ffffffff81010fb5 ffff8805516b5cf0: ffffffff810c99cf 0000000000000010 ffff8805516b5d00: 0000000000000046 ffff8805516b5c70 ffff8805516b5d10: 0000000000000018 0000000000000004 ffff8805516b5d20: ffffffffa051535b ffff8808bb14cf40 ffff8805516b5d30: ffff8805516b5db0 ffffffff8152a2c4 #2 [ffff8805516b5d38] panic at ffffffff8152a2c4 ffff8805516b5d40: ffffffffa0521860 ffffffffa051535b ffff8805516b5d50: ffffffff00000008 ffff8805516b5dc0 ffff8805516b5d60: ffff8805516b5d70 0000000000000000 ffff8805516b5d70: 0000000000343031 0000000000000286 ffff8805516b5d80: ffffffff81aa6460 0000000000000000 ffff8805516b5d90: 0000000000000000 0000000000000000 ffff8805516b5da0: ffffffffa08d3500 ffffffffa08d3500 ffff8805516b5db0: ffff8805516b5dd0 ffffffffa04f2eeb #3 [ffff8805516b5db8] lbug_with_loc at ffffffffa04f2eeb [libcfs] ffff8805516b5dc0: ffff8805516b5e98 0000000000000000 ffff8805516b5dd0: ffff8805516b5df0 ffffffffa082938b #4 [ffff8805516b5dd8] ptlrpc_connection_put at ffffffffa082938b [ptlrpc] ffff8805516b5de0: 0000000000000000 ffff880903ac1000 <=== HERE : %r12 = ffff880903ac1000 ffff8805516b5df0: ffff8805516b5e20 ffffffffa066c70d #5 [ffff8805516b5df8] class_import_destroy at ffffffffa066c70d [obdclass] ffff8805516b5e00: ffff880263925520 0000000000000000 ffff8805516b5e10: ffff880903ac1000 ffff8805516b5e80 <=== HERE : %r12 = ffff880903ac1000 ffff8805516b5e20: ffff8805516b5e60 ffffffffa067067b #6 [ffff8805516b5e28] obd_zombie_impexp_cull at ffffffffa067067b [obdclass] ffff8805516b5e30: ffff8805516b5e60 0000000000000001 ffff8805516b5e40: ffff88086f452040 ffff8805516b5e80 ffff8805516b5e50: ffff88086f452040 ffff88086f452040 ffff8805516b5e60: ffff8805516b5ee0 ffffffffa0670be5 #7 [ffff8805516b5e68] obd_zombie_impexp_thread at ffffffffa0670be5 [obdclass] ffff8805516b5e70: 000000000000ec08 0000000000000000 ffff8805516b5e80: ffff880800000000 ffff88086f452040 ffff8805516b5e90: ffffffff81064bc0 dead000000100100 ffff8805516b5ea0: dead000000200200 0000000000000000 ffff8805516b5eb0: ffff8805516b5ee0 ffff88087927e180 ffff8805516b5ec0: ffff8805516b5ef8 ffffffffa0670b80 ffff8805516b5ed0: 0000000000000000 ffff880479c40ab0 ffff8805516b5ee0: ffff8805516b5f40 ffffffff8109e71e #8 [ffff8805516b5ee8] kthread at ffffffff8109e71e ffff8805516b5ef0: ffffffff00000000 0000000000000000 ffff8805516b5f00: 0000001100000000 ffff8805516b5f08 ffff8805516b5f10: ffff8805516b5f08 0000000000000000 ffff8805516b5f20: ffff88087927e180 ffffffff81ebb4c8 ffff8805516b5f30: ffff880479c40ab0 0000000000000000 ffff8805516b5f40: ffff880479c45f40 ffffffff8100c20a #9 [ffff8805516b5f48] kernel_thread at ffffffff8100c20a crash> p/x 0xffff880903ac1000 + 0x50 $4 = 0xffff880903ac1050 crash> x/a 0xffff880903ac1050 0xffff880903ac1050: 0xffff880b157a7540 crash> struct ptlrpc_connection 0xffff880b157a7540 struct ptlrpc_connection { c_hash = { next = 0xffff880496d62b40, pprev = 0xffff8803242fdd00 }, c_self = 1407422302593066, c_peer = { nid = 1407422302596706, pid = 12345 }, c_remote_uuid = { uuid = "NET_0x5000OBFUSCATE_UUID\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" }, c_refcount = { counter = 3 } } What about the import? 1588 /** 1589 * kill zombie imports and exports 1590 */ 1591 void obd_zombie_impexp_cull(void) 1592 { 1593 struct obd_import *import; 1594 struct obd_export *export; 1595 ENTRY; 1596 1597 do { 1598 spin_lock(&obd_zombie_impexp_lock); 1599 1600 import = NULL; 1601 if (!cfs_list_empty(&obd_zombie_imports)) { 1602 import = cfs_list_entry(obd_zombie_imports.next, 1603 struct obd_import, 1604 imp_zombie_chain); 1605 cfs_list_del_init(&import->imp_zombie_chain); 1606 } 1607 1608 export = NULL; 1609 if (!cfs_list_empty(&obd_zombie_exports)) { 1610 export = cfs_list_entry(obd_zombie_exports.next, 1611 struct obd_export, 1612 exp_obd_chain); 1613 cfs_list_del_init(&export->exp_obd_chain); 1614 } 1615 1616 spin_unlock(&obd_zombie_impexp_lock); 1617 1618 if (import != NULL) { 1619 class_import_destroy(import); 1620 spin_lock(&obd_zombie_impexp_lock); 1621 zombies_count--; 1622 spin_unlock(&obd_zombie_impexp_lock); 1623 } 1624 crash> dis -rl 0xffffffffa067067b [...] 0xffffffffa0670673 : mov %r12,%rdi 0xffffffffa0670676 : callq 0xffffffffa066c6b0 /usr/src/debug/lustre-2.5.3.90/lustre/obdclass/genops.c: 1620 0xffffffffa067067b : mov $0xffffffffa072d530,%rdi import = %r12 = 0xffff880903ac1000 crash> struct obd_import.imp_connection 0xffff880903ac1000 imp_connection = 0xffff880b157a7540 crash> struct ptlrpc_connection 0xffff880b157a7540 struct ptlrpc_connection { c_hash = { next = 0xffff880496d62b40, pprev = 0xffff8803242fdd00 }, c_self = 1407422302593066, c_peer = { nid = 1407422302596706, pid = 12345 }, c_remote_uuid = { uuid = "NET_0x5000OBFUSCATE_UUID\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" }, c_refcount = { counter = 3 } } # AutoView -b 8 -n compute2823 @@@@@@@@@ compute2823 : hwm2823 P3-ZK S6010 @@@@@@@@@ Mon Jun 08 11:03:27 CEST 2015 Analyze: crash blocage lustre User: apr Mon Jun 08 13:28:12 CEST 2015 Software: K0038K NMI 2015 system idle PANIC: "Kernel panic - not syncing: NMI: Not continuing" User: DM Mon Jun 08 13:28:48 CEST 2015 Deploy: @readyToDeploy: all tests succeded User: DM Mon Jun 08 13:42:36 CEST 2015 Unitaire: @readyToUnitaire: all tests succeded User: DM Mon Jun 08 16:50:02 CEST 2015 Unitaire: @readyToUnitaire: all tests succeded User: wr Tue Jun 09 11:20:38 CEST 2015 TestIB: @readyToTestIB: all tests succeded User: wr Tue Jun 09 12:09:05 CEST 2015 Burned: @readyToBurned: all tests succeded User: DM Tue Jun 09 14:13:27 CEST 2015 Prod: remise en prod User: LL What about the LustreError before the LASSERT? <3>LustreError: 12255:0:(sec.c:379:import_sec_validate_get()) import ffff880903ac1000 (FULL) with no sec 365 static int import_sec_validate_get(struct obd_import *imp, 366 struct ptlrpc_sec **sec) 367 { 368 int rc; 369 370 if (unlikely(imp->imp_sec_expire)) { 371 rc = import_sec_check_expire(imp); 372 if (rc) 373 return rc; 374 } 375 376 *sec = sptlrpc_import_sec_ref(imp); 377 if (*sec == NULL) { 378 CERROR("import %p (%s) with no sec\n", 379 imp, ptlrpc_import_state_name(imp->imp_state)); 380 return -EACCES; 381 } 382 383 if (unlikely((*sec)->ps_dying)) { 384 CERROR("attempt to use dying sec %p\n", sec); 385 sptlrpc_sec_put(*sec); 386 return -EACCES; 387 } 388 389 return 0; 390 } This is the same import at 0xffff880903ac1000. 1335 struct ptlrpc_sec *sptlrpc_import_sec_ref(struct obd_import *imp) 1336 { 1337 struct ptlrpc_sec *sec; 1338 1339 spin_lock(&imp->imp_lock); 1340 sec = sptlrpc_sec_get(imp->imp_sec); 1341 spin_unlock(&imp->imp_lock); 1342 1343 return sec; 1344 } 1345 EXPORT_SYMBOL(sptlrpc_import_sec_ref); 1259 struct ptlrpc_sec *sptlrpc_sec_get(struct ptlrpc_sec *sec) 1260 { 1261 if (sec) 1262 cfs_atomic_inc(&sec->ps_refcount); 1263 1264 return sec; 1265 } 1266 EXPORT_SYMBOL(sptlrpc_sec_get); crash> struct -o obd_import.imp_sec struct obd_import { [208] struct ptlrpc_sec *imp_sec; } crash> p/x 0xffff880903ac1000 + 208 $218 = 0xffff880903ac10d0 crash> x/a 0xffff880903ac10d0 0xffff880903ac10d0: 0xffffffffa09138c0 crash> struct ptlrpc_sec 0xffffffffa09138c0 struct ptlrpc_sec { ps_policy = 0xffffffffa08f1be0, ps_refcount = { counter = 4624 }, ps_nctx = { counter = 1 }, ps_id = -1, ps_flvr = { sf_rpc = 0, sf_flags = 0, u_rpc = {}, u_bulk = { hash = { hash_alg = 0 '\000' } } }, ps_part = LUSTRE_SP_MDT, ps_dying = 0, ps_import = 0x0, ps_lock = { raw_lock = { slock = 2860624513 } }, ps_gc_list = { next = 0xffffffffa09138f8, prev = 0xffffffffa09138f8 }, ps_gc_interval = 0, ps_gc_next = 0 } crash> struct ptlrpc_sec_policy 0xffffffffa08f1be0 struct ptlrpc_sec_policy { sp_owner = 0xffffffffa08fc440, sp_name = 0xffffffffa08a9330 "sec.null", sp_policy = 0, sp_cops = 0xffffffffa08f2340, sp_sops = 0xffffffffa08f23c0 } Debug log --- 02000000:00020000:28.0F:1433652063.227347:0:12255:0:(sec.c:379:import_sec_validate_get()) import ffff880903ac1000 (FULL) with no sec 00000100:00040000:28.0:1433652063.248671:0:19842:0:(connection.c:104:ptlrpc_connection_put()) ASSERTION( atomic_read(&conn->c_refcount) > 1 ) failed: 00000100:00040000:28.0:1433652063.259452:0:19842:0:(connection.c:104:ptlrpc_connection_put()) LBUG --- crash> bt 12255 PID: 12255 TASK: ffff88086e317520 CPU: 28 COMMAND: "mdt00_060" #0 [ffff88034c695460] schedule at ffffffff8152a970 #1 [ffff88034c695538] __cond_resched at ffffffff8106c88a #2 [ffff88034c695558] _cond_resched at ffffffff8152b410 #3 [ffff88034c695568] kmem_cache_alloc at ffffffff811751ed #4 [ffff88034c6955a8] ptlrpc_request_cache_alloc at ffffffffa0821beb [ptlrpc] #5 [ffff88034c6955c8] ptlrpc_request_alloc_internal at ffffffffa082294a [ptlrpc] #6 [ffff88034c6955f8] ptlrpc_request_alloc at ffffffffa0822aa3 [ptlrpc] #7 [ffff88034c695608] ptlrpc_request_alloc_pack at ffffffffa0822ad3 [ptlrpc] #8 [ffff88034c695638] ldlm_server_blocking_ast at ffffffffa081230d [ptlrpc] #9 [ffff88034c695688] ldlm_work_bl_ast_lock at ffffffffa07e538d [ptlrpc] #10 [ffff88034c695708] ptlrpc_set_wait at ffffffffa08269cc [ptlrpc] #11 [ffff88034c6957a8] ldlm_run_ast_work at ffffffffa07e80cb [ptlrpc] #12 [ffff88034c6957d8] ldlm_process_inodebits_lock at ffffffffa081693f [ptlrpc] #13 [ffff88034c695858] ldlm_lock_enqueue at ffffffffa07e7613 [ptlrpc] #14 [ffff88034c6958c8] ldlm_cli_enqueue_local at ffffffffa0806e23 [ptlrpc] #15 [ffff88034c695948] mdt_object_lock0 at ffffffffa0d82c14 [mdt] #16 [ffff88034c6959f8] mdt_object_lock at ffffffffa0d83474 [mdt] #17 [ffff88034c695a08] mdt_object_find_lock at ffffffffa0d83631 [mdt] #18 [ffff88034c695a38] mdt_reint_open at ffffffffa0daf92c [mdt] #19 [ffff88034c695b18] mdt_reint_rec at ffffffffa0d98581 [mdt] #20 [ffff88034c695b38] mdt_reint_internal at ffffffffa0d7dee3 [mdt] #21 [ffff88034c695b78] mdt_intent_reint at ffffffffa0d7e46e [mdt] #22 [ffff88034c695bc8] mdt_intent_policy at ffffffffa0d7bc4e [mdt] #23 [ffff88034c695c08] ldlm_lock_enqueue at ffffffffa07e72e5 [ptlrpc] #24 [ffff88034c695c78] ldlm_handle_enqueue0 at ffffffffa0810e8b [ptlrpc] #25 [ffff88034c695ce8] mdt_enqueue at ffffffffa0d7c116 [mdt] #26 [ffff88034c695d08] mdt_handle_common at ffffffffa0d80bfa [mdt] #27 [ffff88034c695d58] mds_regular_handle at ffffffffa0dbd695 [mdt] #28 [ffff88034c695d68] ptlrpc_server_handle_request at ffffffffa083fee5 [ptlrpc] #29 [ffff88034c695e48] ptlrpc_main at ffffffffa084266d [ptlrpc] #30 [ffff88034c695ee8] kthread at ffffffff8109e71e #31 [ffff88034c695f48] kernel_thread at ffffffff8100c20a