[root@germain134 WW.WW.0.1-2012-12-03-19:01:28]# EDITOR=emacs crash /usr/lib/debug/lib/modules/$(uname -r)/vmlinux  vmcore

crash 5.1.1-2.bl6
Copyright (C) 2002-2011  Red Hat, Inc.
Copyright (C) 2004, 2005, 2006  IBM Corporation
Copyright (C) 1999-2006  Hewlett-Packard Co
Copyright (C) 2005, 2006  Fujitsu Limited
Copyright (C) 2006, 2007  VA Linux Systems Japan K.K.
Copyright (C) 2005  NEC Corporation
Copyright (C) 1999, 2002, 2007  Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002  Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions.  Enter "help copying" to see the conditions.
This program has absolutely no warranty.  Enter "help warranty" for details.

GNU gdb (GDB) 7.0
Copyright (C) 2009 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-unknown-linux-gnu"...


      KERNEL: /usr/lib/debug/lib/modules/2.6.32-220.23.1.bl6.Bull.28.8.x86_64/vmlinux
    DUMPFILE: vmcore  [PARTIAL DUMP]
        CPUS: 8
        DATE: Mon Dec  3 19:01:13 2012
      UPTIME: 12 days, 07:56:38
LOAD AVERAGE: 4.98, 4.78, 4.84
       TASKS: 747
    NODENAME: germain134
     RELEASE: 2.6.32-220.23.1.bl6.Bull.28.8.x86_64
     VERSION: #1 SMP Thu Jul 5 16:46:35 CEST 2012
     MACHINE: x86_64  (2800 Mhz)
      MEMORY: 24 GB
       PANIC: ""
         PID: 26203
     COMMAND: "ldlm_bl_44"
        TASK: ffff88033d5dc040  [THREAD_INFO: ffff8802e12e4000]
         CPU: 7
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 26203  TASK: ffff88033d5dc040  CPU: 7   COMMAND: "ldlm_bl_44"
 #0 [ffff8802e12e7ab0] machine_kexec at ffffffff8102895b
 #1 [ffff8802e12e7b10] crash_kexec at ffffffff810a4622
 #2 [ffff8802e12e7be0] oops_end at ffffffff814886c0
 #3 [ffff8802e12e7c10] die at ffffffff8100714b
 #4 [ffff8802e12e7c40] do_general_protection at ffffffff81488252
 #5 [ffff8802e12e7c70] general_protection at ffffffff81487a25
    [exception RIP: d_kill+53]
    RIP: ffffffff81179f05  RSP: ffff8802e12e7d20  RFLAGS: 00010282
    RAX: dead000000200200  RBX: ffff8805d305a180  RCX: ffff8805d305a1c0
    RDX: dead000000100100  RSI: dead000000200200  RDI: dead000000100100
    RBP: ffff8802e12e7d30   R8: 00000000deadbeef   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 5a5a5a5a5a5a5a5a  R12: ffff8805d305a188
    R13: ffffffffa0a848a0  R14: ffff88053dd101e0  R15: ffff8804d9652af8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #6 [ffff8802e12e7d38] dput at ffffffff8117b971
 #7 [ffff8802e12e7d58] ll_drop_dentry at ffffffffa09eb626 [lustre]
 #8 [ffff8802e12e7d98] ll_md_blocking_ast at ffffffffa0a2c702 [lustre]
 #9 [ffff8802e12e7e08] ldlm_cancel_callback at ffffffffa05f01f0 [ptlrpc]
#10 [ffff8802e12e7e28] ldlm_cli_cancel_local at ffffffffa060a20b [ptlrpc]
#11 [ffff8802e12e7e58] ldlm_cli_cancel_list_local at ffffffffa060cafd [ptlrpc]
#12 [ffff8802e12e7eb8] ldlm_bl_thread_main at ffffffffa06114f0 [ptlrpc]
#13 [ffff8802e12e7f48] kernel_thread at ffffffff8100412a
crash> ps | grep -v " IN "
   PID    PPID  CPU       TASK        ST  %MEM     VSZ    RSS  COMM
      0      0   0  ffffffff81a0e020  RU   0.0       0      0  [swapper]
>     0      0   1  ffff88063e4ee040  RU   0.0       0      0  [swapper]
      0      0   2  ffff88063e4ee790  RU   0.0       0      0  [swapper]
>     0      0   3  ffff88063e4f6080  RU   0.0       0      0  [swapper]
>     0      0   4  ffff88063e4f67d0  RU   0.0       0      0  [swapper]
>     0      0   5  ffff88063e5770c0  RU   0.0       0      0  [swapper]
>     0      0   6  ffff88063e577810  RU   0.0       0      0  [swapper]
      0      0   7  ffff88063e5df100  RU   0.0       0      0  [swapper]
      3      2   0  ffff88033e4fd7d0  RU   0.0       0      0  [migration/0]
> 10428  10427   0  ffff88033d2c1850  RU   0.0  105164    776  umount
> 13878  13805   2  ffff88063c92f080  RU   0.6 10660280 167836  java
> 26203      2   7  ffff88033d5dc040  RU   0.0       0      0  [ldlm_bl_44]
  28936      1   6  ffff88063be6d810  UN   0.0   13428   1136  lctl
crash> bt 10428
PID: 10428  TASK: ffff88033d2c1850  CPU: 0   COMMAND: "umount"
 #0 [ffff880028207e90] crash_nmi_callback at ffffffff81020b56
 #1 [ffff880028207ea0] notifier_call_chain at ffffffff8148a755
 #2 [ffff880028207ee0] atomic_notifier_call_chain at ffffffff8148a7ba
 #3 [ffff880028207ef0] notify_die at ffffffff810817ce
 #4 [ffff880028207f20] do_nmi at ffffffff814883d3
 #5 [ffff880028207f50] nmi at ffffffff81487ce0
    [exception RIP: ldlm_res_hop_fid_hash+25]
    RIP: ffffffffa05f43b9  RSP: ffff880120b2bb98  RFLAGS: 00000292
    RAX: ffffffffa06acda0  RBX: ffff8805f7cbc300  RCX: 0000000000000010
    RDX: 000000000000ffff  RSI: ffff880120b2bcc8  RDI: ffff8805f7cbc300
    RBP: ffff880120b2bbd8   R8: ffff880120b2bc48   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 00000002000abc55  R12: 0000000000000010
    R13: ffff8805f8216bc0  R14: ffff880120b2bc48  R15: 0000000000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
--- <NMI exception stack> ---
 #6 [ffff880120b2bb98] ldlm_res_hop_fid_hash at ffffffffa05f43b9 [ptlrpc]
 #7 [ffff880120b2bbe0] cfs_hash_bd_from_key at ffffffffa03db192 [libcfs]
 #8 [ffff880120b2bc10] cfs_hash_bd_get at ffffffffa03db8a5 [libcfs]
 #9 [ffff880120b2bc20] ldlm_resource_get at ffffffffa05f7099 [ptlrpc]
#10 [ffff880120b2bc90] ldlm_resource_iterate at ffffffffa0609b1d [ptlrpc]
#11 [ffff880120b2bcc0] mdc_find_cbdata at ffffffffa08a63cc [mdc]
#12 [ffff880120b2bd20] lmv_find_cbdata at ffffffffa0b17501 [lmv]
#13 [ffff880120b2bd80] find_cbdata at ffffffffa09ecdd2 [lustre]
#14 [ffff880120b2bdb0] ll_d_iput at ffffffffa09ed8be [lustre]
#15 [ffff880120b2bdd0] shrink_dcache_for_umount_subtree at ffffffff8117b36d
#16 [ffff880120b2be20] shrink_dcache_for_umount at ffffffff8117b4f6
#17 [ffff880120b2be40] generic_shutdown_super at ffffffff811653ef
#18 [ffff880120b2be60] kill_anon_super at ffffffff81165546
#19 [ffff880120b2be80] lustre_kill_super at ffffffffa04c699a [obdclass]
#20 [ffff880120b2bea0] deactivate_super at ffffffff811664e0
#21 [ffff880120b2bec0] mntput_no_expire at ffffffff811826bf
#22 [ffff880120b2bef0] sys_umount at ffffffff81183188
#23 [ffff880120b2bf80] system_call_fastpath at ffffffff810030f2
    RIP: 00007fd4f64ccd67  RSP: 00007fff034b8a38  RFLAGS: 00010202
    RAX: 00000000000000a6  RBX: ffffffff810030f2  RCX: 0000000000000010
    RDX: 0000000000000000  RSI: 0000000000000000  RDI: 00007fd4f82f8f80
    RBP: 00007fd4f82f8f40   R8: 00007fd4f82f8fa0   R9: 0000000000000000
    R10: 00007fff034b8860  R11: 0000000000000246  R12: 0000000000000000
    R13: 0000000000000000  R14: 0000000000000000  R15: 00007fd4f82f8fe0
    ORIG_RAX: 00000000000000a6  CS: 0033  SS: 002b
crash> ps -l
[1063346611036521]  PID: 23     TASK: ffff88033e5c5850  CPU: 6   COMMAND: "ksoftirqd/6"
[1063346580359790]  PID: 5789   TASK: ffff88033d66d0c0  CPU: 5   COMMAND: "robinhood"
[1063346578164264]  PID: 13857  TASK: ffff88033d090790  CPU: 5   COMMAND: "java"
[1063346538406674]  PID: 13817  TASK: ffff88063e60f810  CPU: 5   COMMAND: "java"
[1063346302681733]  PID: 26545  TASK: ffff8802df5d2100  CPU: 5   COMMAND: "ldlm_poold"
[1063346272323389]  PID: 24252  TASK: ffff88063c9967d0  CPU: 3   COMMAND: "router_checker"
[1063346266575091]  PID: 13577  TASK: ffff88063cbf60c0  CPU: 1   COMMAND: "nrpe"
[1063346122811200]  PID: 12593  TASK: ffff88033c3ad0c0  CPU: 4   COMMAND: "mlx4_sense"
[1063346113262074]  PID: 25411  TASK: ffff88063d664790  CPU: 6   COMMAND: "ptlrpcd_7"
[1063346113252422]  PID: 25409  TASK: ffff88063c621080  CPU: 6   COMMAND: "ptlrpcd_5"
[1063346112658544]  PID: 25403  TASK: ffff88063d665810  CPU: 3   COMMAND: "ptlrpcd_0"
[1063346112656454]  PID: 25406  TASK: ffff88063d7e17d0  CPU: 3   COMMAND: "ptlrpcd_2"
[1063346112654119]  PID: 25405  TASK: ffff88063cd407d0  CPU: 3   COMMAND: "ptlrpcd_1"
[1063346112651766]  PID: 25408  TASK: ffff88063cd40080  CPU: 3   COMMAND: "ptlrpcd_4"
[1063346112648800]  PID: 25410  TASK: ffff88063c996080  CPU: 3   COMMAND: "ptlrpcd_6"
[1063346112631517]  PID: 25407  TASK: ffff88060f28a810  CPU: 3   COMMAND: "ptlrpcd_3"
[1063346103272347]  PID: 33     TASK: ffff88033e60e790  CPU: 6   COMMAND: "events/6"
[1063346013706210]  PID: 10428  TASK: ffff88033d2c1850  CPU: 0   COMMAND: "umount"
[1063346012143635]  PID: 27     TASK: ffff88033e5e37d0  CPU: 0   COMMAND: "events/0"
[1063345983297194]  PID: 25402  TASK: ffff88063d126790  CPU: 5   COMMAND: "ptlrpcd_rcv"
[1063345982464599]  PID: 28     TASK: ffff88033e5e3080  CPU: 1   COMMAND: "events/1"
[1063345958892454]  PID: 13878  TASK: ffff88063c92f080  CPU: 2   COMMAND: "java"
[1063345952674108]  PID: 29     TASK: ffff88033e60b810  CPU: 2   COMMAND: "events/2"
[1063345922996817]  PID: 30     TASK: ffff88033e60b0c0  CPU: 3   COMMAND: "events/3"
[1063345915539800]  PID: 1263   TASK: ffff88033c33d790  CPU: 5   COMMAND: "jbd2/sda3-8"
[1063345915524160]  PID: 55     TASK: ffff88033e620850  CPU: 5   COMMAND: "kblockd/5"
[1063345901495091]  PID: 50     TASK: ffff88033e61c040  CPU: 0   COMMAND: "kblockd/0"
[1063345893267955]  PID: 31     TASK: ffff88033e60c850  CPU: 4   COMMAND: "events/4"
crash> dmesg
Initializing cgroup subsys cpuset
Initializing cgroup subsys cpu
Linux version 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 (efix@atlas.frec.bull.fr) (gcc version 4.4.5 20110214 (Bull 4.4.5-6) (GCC) ) #1 SMP Thu Jul 5 16:46:35 CEST 2012
Command line: ro root=UUID=4f2412a7-7d85-442c-8793-cc5a63bf89c5 rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=C SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us console=tty0 console=ttyS1,115200 crashkernel=192M rdloaddriver=ahci rdloaddriver=aacraid rdloaddriver=megaraid_sas cgroup_disable=memory selinux=0 nmi_watchdog=0 rdblacklist=lpfc transparent_hugepage=never pciehp_disable tick_synchro=0 audit=0
KERNEL supported cpus:
  Intel GenuineIntel
  AMD AuthenticAMD
  Centaur CentaurHauls
......

Lustre: DEBUG MARKER: Wed Nov 21 18:45:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
Lustre: DEBUG MARKER: Wed Nov 21 18:50:01 2012

Lustre: DEBUG MARKER: Wed Nov 21 18:55:01 2012

....

CE: hpet increasing min_delta_ns to 60338 nsec
Lustre: DEBUG MARKER: Thu Nov 22 14:50:01 2012

.......
Lustre: DEBUG MARKER: Thu Nov 22 19:05:01 2012

CE: hpet increasing min_delta_ns to 90506 nsec
Lustre: DEBUG MARKER: Thu Nov 22 19:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 22 19:25:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
Lustre: DEBUG MARKER: Thu Nov 22 19:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 22 19:45:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 22 19:50:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 15 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 22 19:55:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 19 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 22 20:00:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 11 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 22 20:05:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 27 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 22 20:10:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 08:05:01 2012

........
Lustre: DEBUG MARKER: Fri Nov 23 08:10:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 81 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 08:15:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 15 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 08:20:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 08:25:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 08:30:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 15 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 08:35:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 08:40:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 35 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 08:45:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 08:50:01 2012

.....

Lustre: DEBUG MARKER: Fri Nov 23 21:00:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 23 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 21:05:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 21:10:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 21:15:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 32 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 21:20:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 21:25:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 44 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 21:30:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 21:35:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 541 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 23 21:40:01 2012

Lustre: DEBUG MARKER: Fri Nov 23 21:45:01 2012

.........

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 60 previous similar messages
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 11 previous similar messages
Lustre: DEBUG MARKER: Sat Nov 24 09:50:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
Lustre: DEBUG MARKER: Sat Nov 24 09:55:01 2012

Lustre: DEBUG MARKER: Sat Nov 24 10:00:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
Lustre: DEBUG MARKER: Sat Nov 24 10:05:01 2012

Lustre: DEBUG MARKER: Sat Nov 24 10:10:01 2012

LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 3966:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 27 previous similar messages
Lustre: DEBUG MARKER: Sat Nov 24 10:15:01 2012

.........

CE: hpet increasing min_delta_ns to 135758 nsec
Lustre: DEBUG MARKER: Sun Nov 25 14:30:02 2012

.......

Lustre: DEBUG MARKER: Mon Nov 26 10:25:01 2012

Lustre: 20801:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: DEBUG MARKER: Mon Nov 26 10:30:01 2012

Lustre: 29723:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 29723:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: 29747:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 29747:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: Mon Nov 26 10:35:02 2012

Lustre: 29025:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 29025:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: Mon Nov 26 10:40:01 2012

Lustre: 11088:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 11088:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: Mon Nov 26 10:45:01 2012

Lustre: 11972:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 11972:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: 12106:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 12106:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: 15115:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 15115:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: Mon Nov 26 10:50:01 2012

Lustre: 19981:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 19981:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 3 previous similar messages
Lustre: 22999:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 22999:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: Mon Nov 26 10:55:01 2012

Lustre: 27693:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) message send failed (-32)
Lustre: 27693:0:(kernel_user_comm.c:177:libcfs_kkuc_msg_put()) Skipped 3 previous similar messages
Lustre: DEBUG MARKER: Mon Nov 26 11:00:01 2012

.....

Lustre: DEBUG MARKER: Mon Nov 26 11:35:01 2012

LustreError: 2371:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 2371:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 15 previous similar messages
Lustre: DEBUG MARKER: Mon Nov 26 11:40:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 11:45:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 11:50:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 11:55:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 12:00:02 2012

Lustre: DEBUG MARKER: Mon Nov 26 12:05:01 2012

LustreError: 2371:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 2371:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 507 previous similar messages
Lustre: DEBUG MARKER: Mon Nov 26 12:10:01 2012

LustreError: 2371:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 2371:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: Mon Nov 26 12:15:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 12:20:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 12:25:01 2012

Lustre: DEBUG MARKER: Mon Nov 26 12:30:01 2012

LustreError: 11-0: an error occurred while communicating with XX.XX.160.41@o2ib. The ldlm_enqueue operation failed with -107
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection to ptmp-MDT0000 (at XX.XX.160.41@o2ib) was lost; in progress operations using this service will wait for recovery to complete
LustreError: 167-0: This client was evicted by ptmp-MDT0000; in progress operations using this service will fail.
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -4
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8805ecace800 x1419262450292389/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 568/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8805ecace800 x1419262450292393/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 568/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 3 previous similar messages
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 4 previous similar messages
LustreError: 5818:0:(file.c:158:ll_close_inode_openhandle()) inode 144126988616039760 mdc close failed: rc = -108
LustreError: Skipped 1 previous similar message
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88035724c800 x1419262450294146/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 584/1104 e 0 to 0 dl 0 ref 2 fl Rpc:P/0/ffffffff rc 0/-1
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 1752 previous similar messages
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 1743 previous similar messages
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88029e653000 x1419262450297869/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 576/1104 e 0 to 0 dl 0 ref 2 fl Rpc:P/0/ffffffff rc 0/-1
LustreError: 5818:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 3722 previous similar messages
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 5818:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 3605 previous similar messages
LustreError: 5806:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8803556dd000 x1419262450301885/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 560/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 5806:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 4009 previous similar messages
LustreError: 5806:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 5806:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 3781 previous similar messages
LustreError: 2354:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8802a881f000 x1419262450305926/t0(0) o46->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 232/192 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 2354:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 4034 previous similar messages
LustreError: 2352:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 2352:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 3729 previous similar messages
LustreError: 2351:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88029e45ec00 x1419262450307129/t0(0) o46->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 232/192 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 2351:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 400 previous similar messages
LustreError: 2351:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 2351:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 205 previous similar messages
LustreError: 15234:0:(mdc_request.c:1300:mdc_changelog_send_thread()) llog_create() failed -108
LustreError: 2348:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88025c0a7c00 x1419262450308418/t0(0) o46->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 232/192 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 2348:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 1264 previous similar messages
LustreError: 2347:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 2347:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 429 previous similar messages
LustreError: 2351:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8804912b9c00 x1419262450315233/t0(0) o46->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 232/192 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 2351:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 5044 previous similar messages
LustreError: 2349:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 2349:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 2670 previous similar messages
Lustre: DEBUG MARKER: Mon Nov 26 12:35:01 2012

LustreError: 2345:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff880599fc1000 x1419262450322686/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 568/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 2345:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 4964 previous similar messages
LustreError: 2354:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 2354:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 1583 previous similar messages
LustreError: 26271:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88029e7b1000 x1419262450334089/t0(0) o35->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:23/10 lens 360/2056 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 26271:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 4334 previous similar messages
LustreError: 26271:0:(file.c:158:ll_close_inode_openhandle()) inode 144115188109410306 mdc close failed: rc = -108
Lustre: DEBUG MARKER: Mon Nov 26 12:40:02 2012

LustreError: 21495:0:(lmv_obd.c:1200:lmv_statfs()) can't stat MDS #0 (ptmp-MDT0000-mdc-ffff88033cd03000), error -107
LustreError: 21495:0:(llite_lib.c:1444:ll_statfs_internal()) md_statfs fails: rc = -107
LustreError: 13792:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 13792:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff880093243580 (8590405110/53354/0/0) (rc: 1)
LustreError: 13792:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 13792:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff8805cfc07d40 (8590404630/53354/0/0) (rc: 1)
LustreError: 13792:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 13792:0:(ldlm_resource.c:749:ldlm_resource_complain()) Skipped 255343 previous similar messages
LustreError: 13792:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff880422b97300 (8590402881/101514/0/0) (rc: 1)
LustreError: 13792:0:(ldlm_resource.c:755:ldlm_resource_complain()) Skipped 255343 previous similar messages
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection restored to ptmp-MDT0000 (at XX.XX.160.41@o2ib)
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection to ptmp-MDT0000 (at XX.XX.160.41@o2ib) was lost; in progress operations using this service will wait for recovery to complete
LustreError: 26284:0:(ldlm_request.c:1172:ldlm_cli_cancel_req()) Got rc -107 from cancel RPC: canceling anyway
LustreError: 26284:0:(ldlm_request.c:1799:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -107
LustreError: 167-0: This client was evicted by ptmp-MDT0000; in progress operations using this service will fail.
LustreError: 27294:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 27294:0:(ldlm_resource.c:749:ldlm_resource_complain()) Skipped 476723 previous similar messages
LustreError: 27294:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff880473dd3e80 (8590637791/114790/0/0) (rc: 0)
LustreError: 27294:0:(ldlm_resource.c:755:ldlm_resource_complain()) Skipped 476723 previous similar messages
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection restored to ptmp-MDT0000 (at XX.XX.160.41@o2ib)
Lustre: DEBUG MARKER: Mon Nov 26 12:45:02 2012

Lustre: DEBUG MARKER: Mon Nov 26 12:50:01 2012

........

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 22 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 39 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 81 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 152 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 283 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 642 previous similar messages
Lustre: DEBUG MARKER: Tue Nov 27 14:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 1162 previous similar messages
Lustre: DEBUG MARKER: Tue Nov 27 15:00:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2234 previous similar messages
Lustre: DEBUG MARKER: Tue Nov 27 15:05:01 2012

..........

Lustre: DEBUG MARKER: Wed Nov 28 03:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 1207 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 606 previous similar messages
Lustre: DEBUG MARKER: Wed Nov 28 03:20:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 964 previous similar messages
Lustre: DEBUG MARKER: Wed Nov 28 03:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2433 previous similar messages
Lustre: DEBUG MARKER: Wed Nov 28 03:30:02 2012

.......

Lustre: DEBUG MARKER: Wed Nov 28 15:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2198 previous similar messages
Lustre: DEBUG MARKER: Wed Nov 28 15:40:02 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
Lustre: DEBUG MARKER: Wed Nov 28 15:45:01 2012

..........

Lustre: DEBUG MARKER: Thu Nov 29 16:00:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 7 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 17 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:10:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 915 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2071 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 16:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4548 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 16:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4262 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 16:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4389 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 16:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 16:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4232 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 17:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 17:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4332 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 17:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 17:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4391 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 17:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 17:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4247 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 17:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 17:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4428 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 17:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 17:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4242 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 17:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 17:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4570 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 18:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 18:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4348 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 18:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 18:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4445 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 18:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 18:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4223 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 18:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 18:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4301 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 18:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 18:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4168 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 18:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 18:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4555 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 19:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 19:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4228 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 19:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 19:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4257 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 19:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 19:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4453 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 19:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 19:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4150 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 19:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 19:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4422 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 19:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 19:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4093 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 20:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 20:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4159 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 20:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 20:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4256 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 20:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 20:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4214 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 20:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 20:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4266 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 20:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 20:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4031 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 20:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 20:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4127 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 21:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 21:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4340 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 21:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 21:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4301 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 21:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 21:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4301 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 21:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 21:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4212 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 21:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 21:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3976 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 21:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 21:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 3989 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 22:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 22:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4300 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 22:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 22:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4433 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 22:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 22:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4226 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 22:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 22:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4455 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 22:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 22:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4190 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 22:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 22:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4438 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 23:00:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 23:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4208 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 23:10:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 23:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4489 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 23:20:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 23:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4096 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 23:30:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 23:35:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4205 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 23:40:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 23:45:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4386 previous similar messages
Lustre: DEBUG MARKER: Thu Nov 29 23:50:01 2012

Lustre: DEBUG MARKER: Thu Nov 29 23:55:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4425 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 30 00:00:01 2012

Lustre: DEBUG MARKER: Fri Nov 30 00:05:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 4200 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 30 00:10:01 2012

........ 

Lustre: DEBUG MARKER: Fri Nov 30 12:15:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2211 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
Lustre: DEBUG MARKER: Fri Nov 30 12:20:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 101 previous similar messages
Lustre: DEBUG MARKER: Fri Nov 30 12:25:01 2012

Lustre: DEBUG MARKER: Fri Nov 30 12:30:01 2012

...........
Lustre: DEBUG MARKER: Sat Dec  1 00:25:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
Lustre: DEBUG MARKER: Sat Dec  1 00:30:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
Lustre: DEBUG MARKER: Sat Dec  1 00:35:01 2012

...........

Lustre: DEBUG MARKER: Mon Dec  3 13:20:01 2012

LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 35 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 2 previous similar messages
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) error: iocontrol MDC ptmp-MDT0000_UUID on MDTidx 0 cmd c0086696: err = -2
LustreError: 15529:0:(lmv_obd.c:855:lmv_iocontrol()) Skipped 46 previous similar messages
Lustre: DEBUG MARKER: Mon Dec  3 13:25:01 2012

Lustre: DEBUG MARKER: Mon Dec  3 13:30:03 2012

Lustre: DEBUG MARKER: Mon Dec  3 13:35:04 2012

LustreError: 11-0: an error occurred while communicating with XX.XX.160.41@o2ib. The mds_get_info operation failed with -107
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection to ptmp-MDT0000 (at XX.XX.160.41@o2ib) was lost; in progress operations using this service will wait for recovery to complete
LustreError: 167-0: This client was evicted by ptmp-MDT0000; in progress operations using this service will fail.
LustreError: 15529:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88055cd7ec00 x1419262518485321/t0(0) o53->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 256/1248 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 15529:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 1 previous similar message
LustreError: 15515:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 15515:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 797 previous similar messages
LustreError: 15504:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 15504:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 395 previous similar messages
LustreError: 8890:0:(mdc_request.c:1300:mdc_changelog_send_thread()) llog_create() failed -108
LustreError: 15505:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8802a5882800 x1419262518487855/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 568/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 15505:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 1675 previous similar messages
LustreError: 15510:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 15510:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 972 previous similar messages
LustreError: 15505:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff88050e462800 x1419262518494101/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 568/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 15505:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 4533 previous similar messages
LustreError: 9262:0:(lmv_obd.c:1200:lmv_statfs()) can't stat MDS #0 (ptmp-MDT0000-mdc-ffff88033cd03000), error -108
LustreError: 9262:0:(llite_lib.c:1444:ll_statfs_internal()) md_statfs fails: rc = -108
LustreError: 15505:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 15505:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 1847 previous similar messages
LustreError: 15508:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15508:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15508:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 1 previous similar message
LustreError: 15508:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15508:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 148 previous similar messages
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 133 previous similar messages
LustreError: 15506:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15506:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 272 previous similar messages
LustreError: 15505:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15505:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 596 previous similar messages
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 1306 previous similar messages
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 2422 previous similar messages
LustreError: 15529:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID  req@ffff8805d2245400 x1419262518514405/t0(0) o101->ptmp-MDT0000-mdc-ffff88033cd03000@XX.XX.160.41@o2ib:12/10 lens 544/1544 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1
LustreError: 15529:0:(client.c:1076:ptlrpc_import_delay_req()) Skipped 15226 previous similar messages
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 5287 previous similar messages
Lustre: DEBUG MARKER: Mon Dec  3 13:40:03 2012

LustreError: 15529:0:(mdc_locks.c:723:mdc_enqueue()) ldlm_cli_enqueue: -108
LustreError: 15529:0:(mdc_locks.c:723:mdc_enqueue()) Skipped 12001 previous similar messages
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) failure -108 inode 34603009
LustreError: 15529:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 471 previous similar messages
LustreError: 8370:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 8370:0:(ldlm_resource.c:749:ldlm_resource_complain()) Skipped 19976 previous similar messages
LustreError: 8370:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff8805f99f60c0 (8590637713/12367/0/0) (rc: 1)
LustreError: 8370:0:(ldlm_resource.c:755:ldlm_resource_complain()) Skipped 19976 previous similar messages
LustreError: 8370:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 8370:0:(ldlm_resource.c:749:ldlm_resource_complain()) Skipped 118981 previous similar messages
LustreError: 8370:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff8801a31baac0 (8590637713/7515/0/0) (rc: 1)
LustreError: 8370:0:(ldlm_resource.c:755:ldlm_resource_complain()) Skipped 118981 previous similar messages
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection restored to ptmp-MDT0000 (at XX.XX.160.41@o2ib)
LustreError: 11-0: an error occurred while communicating with XX.XX.160.41@o2ib. The ldlm_enqueue operation failed with -107
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection to ptmp-MDT0000 (at XX.XX.160.41@o2ib) was lost; in progress operations using this service will wait for recovery to complete
LustreError: 167-0: This client was evicted by ptmp-MDT0000; in progress operations using this service will fail.
LustreError: 15842:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace ptmp-MDT0000-mdc-ffff88033cd03000 resource refcount nonzero (1) after lock cleanup; forcing cleanup.
LustreError: 15842:0:(ldlm_resource.c:749:ldlm_resource_complain()) Skipped 186379 previous similar messages
LustreError: 15842:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff8805f99f60c0 (8590637713/12367/0/0) (rc: 1)
LustreError: 15842:0:(ldlm_resource.c:755:ldlm_resource_complain()) Skipped 186379 previous similar messages
Lustre: ptmp-MDT0000-mdc-ffff88033cd03000: Connection restored to ptmp-MDT0000 (at XX.XX.160.41@o2ib)
Lustre: DEBUG MARKER: Mon Dec  3 13:45:04 2012

..........

Lustre: DEBUG MARKER: Mon Dec  3 18:20:05 2012

LustreError: 10979:0:(ldlm_request.c:1172:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway
LustreError: 10979:0:(ldlm_request.c:1799:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108
LustreError: 10979:0:(ldlm_request.c:1172:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway
LustreError: 10979:0:(ldlm_request.c:1799:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108
Lustre: Unmounted cdep4-client
Lustre: DEBUG MARKER: Mon Dec  3 18:25:06 2012

......

Lustre: DEBUG MARKER: Mon Dec  3 19:00:05 2012

general protection fault: 0000 [#1] SMP
last sysfs file: /sys/devices/pci0000:00/0000:00:1f.2/host5/scsi_host/host5/proc_name
CPU 7
Modules linked in: lmv(U) mgc(U) lustre(U) lov(U) osc(U) mdc(U) lquota(U) fid(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) lvfs(U) libcfs(U) iptable_filter ip_tables nfs lockd fscache nfs_acl auth_rpcgss sunrpc rdma_ucm(U) rdma_cm(U) iw_cm(U) ib_addr(U) ib_ipoib(U) ib_cm(U) ib_sa(U) ipv6 ib_uverbs(U) ib_umad(U) mlx4_ib(U) ib_mad(U) ib_core(U) mlx4_core(U) dm_mirror dm_region_hash dm_log usbhid hid sg igb dca i2c_i801 i2c_core iTCO_wdt iTCO_vendor_support ehci_hcd uhci_hcd shpchp ext4 mbcache jbd2 sd_mod crc_t10dif dm_mod megaraid_sas aacraid ahci [last unloaded: ipmi_msghandler]

Pid: 26203, comm: ldlm_bl_44 Not tainted 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 #1 INSYDE Thurley/NCB
RIP: 0010:[<ffffffff81179f05>]  [<ffffffff81179f05>] d_kill+0x35/0x80
RSP: 0018:ffff8802e12e7d20  EFLAGS: 00010282
RAX: dead000000200200 RBX: ffff8805d305a180 RCX: ffff8805d305a1c0
RDX: dead000000100100 RSI: dead000000200200 RDI: dead000000100100
RBP: ffff8802e12e7d30 R08: 00000000deadbeef R09: 5a5a5a5a5a5a5a5a
R10: 5a5a5a5a5a5a5a5a R11: 5a5a5a5a5a5a5a5a R12: ffff8805d305a188
R13: ffffffffa0a848a0 R14: ffff88053dd101e0 R15: ffff8804d9652af8
FS:  00007f4b5672d700(0000) GS:ffff88034ac60000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007ff57f8c3000 CR3: 0000000001a06000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process ldlm_bl_44 (pid: 26203, threadinfo ffff8802e12e4000, task ffff88033d5dc040)
Stack:
 ffff8805d305a180 ffff8805d305a188 ffff8802e12e7d50 ffffffff8117b971
<0> ffff8805d305a180 ffffffff81a052c0 ffff8802e12e7d90 ffffffffa09eb626
<0> ffffffff81a052c0 ffff880044892bc0 ffffffffffffff02 ffff8804d9652b38
Call Trace:
 [<ffffffff8117b971>] dput+0xa1/0x190
 [<ffffffffa09eb626>] ll_drop_dentry+0x116/0x2b0 [lustre]
 [<ffffffffa0a2c702>] ll_md_blocking_ast+0x522/0x5f0 [lustre]
 [<ffffffffa04b7fcc>] ? class_handle_unhash+0x3c/0x50 [obdclass]
 [<ffffffffa05f01f0>] ldlm_cancel_callback+0x60/0x100 [ptlrpc]
 [<ffffffffa060a20b>] ldlm_cli_cancel_local+0x7b/0x380 [ptlrpc]
 [<ffffffffa060cafd>] ldlm_cli_cancel_list_local+0xed/0x200 [ptlrpc]
 [<ffffffffa06114f0>] ldlm_bl_thread_main+0x100/0x3d0 [ptlrpc]
 [<ffffffff8104a170>] ? default_wake_function+0x0/0x20
 [<ffffffffa06113f0>] ? ldlm_bl_thread_main+0x0/0x3d0 [ptlrpc]
 [<ffffffff8100412a>] child_rip+0xa/0x20
 [<ffffffffa06113f0>] ? ldlm_bl_thread_main+0x0/0x3d0 [ptlrpc]
 [<ffffffffa06113f0>] ? ldlm_bl_thread_main+0x0/0x3d0 [ptlrpc]
 [<ffffffff81004120>] ? child_rip+0x0/0x20
Code: 1c 24 4c 89 64 24 08 0f 1f 44 00 00 48 8b 57 50 48 8b 47 58 48 be 00 02 20 00 00 00 ad de 48 89 fb 48 bf 00 01 10 00 00 00 ad de <48> 89 42 08 48 89 10 48 89 7b 50 48 89 73 58 48 89 df 83 2d 42
RIP  [<ffffffff81179f05>] d_kill+0x35/0x80
 RSP <ffff8802e12e7d20>
crash> sys
      KERNEL: /usr/lib/debug/lib/modules/2.6.32-220.23.1.bl6.Bull.28.8.x86_64/vmlinux
    DUMPFILE: vmcore  [PARTIAL DUMP]
        CPUS: 8
        DATE: Mon Dec  3 19:01:13 2012
      UPTIME: 12 days, 07:56:38
LOAD AVERAGE: 4.98, 4.78, 4.84
       TASKS: 747
    NODENAME: germain134
     RELEASE: 2.6.32-220.23.1.bl6.Bull.28.8.x86_64
     VERSION: #1 SMP Thu Jul 5 16:46:35 CEST 2012
     MACHINE: x86_64  (2800 Mhz)
      MEMORY: 24 GB
       PANIC: ""
crash> bt
PID: 26203  TASK: ffff88033d5dc040  CPU: 7   COMMAND: "ldlm_bl_44"
 #0 [ffff8802e12e7ab0] machine_kexec at ffffffff8102895b
 #1 [ffff8802e12e7b10] crash_kexec at ffffffff810a4622
 #2 [ffff8802e12e7be0] oops_end at ffffffff814886c0
 #3 [ffff8802e12e7c10] die at ffffffff8100714b
 #4 [ffff8802e12e7c40] do_general_protection at ffffffff81488252
 #5 [ffff8802e12e7c70] general_protection at ffffffff81487a25
    [exception RIP: d_kill+53]
    RIP: ffffffff81179f05  RSP: ffff8802e12e7d20  RFLAGS: 00010282
    RAX: dead000000200200  RBX: ffff8805d305a180  RCX: ffff8805d305a1c0
    RDX: dead000000100100  RSI: dead000000200200  RDI: dead000000100100
    RBP: ffff8802e12e7d30   R8: 00000000deadbeef   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 5a5a5a5a5a5a5a5a  R12: ffff8805d305a188
    R13: ffffffffa0a848a0  R14: ffff88053dd101e0  R15: ffff8804d9652af8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #6 [ffff8802e12e7d38] dput at ffffffff8117b971
 #7 [ffff8802e12e7d58] ll_drop_dentry at ffffffffa09eb626 [lustre]
 #8 [ffff8802e12e7d98] ll_md_blocking_ast at ffffffffa0a2c702 [lustre]
 #9 [ffff8802e12e7e08] ldlm_cancel_callback at ffffffffa05f01f0 [ptlrpc]
#10 [ffff8802e12e7e28] ldlm_cli_cancel_local at ffffffffa060a20b [ptlrpc]
#11 [ffff8802e12e7e58] ldlm_cli_cancel_list_local at ffffffffa060cafd [ptlrpc]
#12 [ffff8802e12e7eb8] ldlm_bl_thread_main at ffffffffa06114f0 [ptlrpc]
#13 [ffff8802e12e7f48] kernel_thread at ffffffff8100412a
crash>



crash> disas d_kill
Dump of assembler code for function d_kill:
0xffffffff81179ed0 <d_kill+0>:  push   %rbp
0xffffffff81179ed1 <d_kill+1>:  mov    %rsp,%rbp
0xffffffff81179ed4 <d_kill+4>:  sub    $0x10,%rsp
0xffffffff81179ed8 <d_kill+8>:  mov    %rbx,(%rsp)
0xffffffff81179edc <d_kill+12>: mov    %r12,0x8(%rsp)
0xffffffff81179ee1 <d_kill+17>: nopl   0x0(%rax,%rax,1)
0xffffffff81179ee6 <d_kill+22>: mov    0x50(%rdi),%rdx     <--- rdi = 0xffff8805d305a180=rbx  struct dentry.d_u.d_child.next
0xffffffff81179eea <d_kill+26>: mov    0x58(%rdi),%rax     <----                              struct dentry.d_u.d_child.prev
0xffffffff81179eee <d_kill+30>: mov    $0xdead000000200200,%rsi
0xffffffff81179ef8 <d_kill+40>: mov    %rdi,%rbx
0xffffffff81179efb <d_kill+43>: mov    $0xdead000000100100,%rdi
0xffffffff81179f05 <d_kill+53>: mov    %rax,0x8(%rdx)      <--- ICI RDX: dead000000100100 n'est pas un pointeur
0xffffffff81179f09 <d_kill+57>: mov    %rdx,(%rax)
0xffffffff81179f0c <d_kill+60>: mov    %rdi,0x50(%rbx)
0xffffffff81179f10 <d_kill+64>: mov    %rsi,0x58(%rbx)
0xffffffff81179f14 <d_kill+68>: mov    %rbx,%rdi
0xffffffff81179f17 <d_kill+71>: subl   $0x1,0x8b9d42(%rip)        # 0xffffffff81a33c60   dentry_stat.nr_dentry--;
0xffffffff81179f1e <d_kill+78>: callq  0xffffffff81179d10 <dentry_iput>
0xffffffff81179f23 <d_kill+83>: mov    0x28(%rbx),%r12
0xffffffff81179f27 <d_kill+87>: mov    $0x0,%eax
0xffffffff81179f2c <d_kill+92>: mov    %rbx,%rdi
0xffffffff81179f2f <d_kill+95>: cmp    %rbx,%r12
0xffffffff81179f32 <d_kill+98>: cmove  %rax,%r12
0xffffffff81179f36 <d_kill+102>:        callq  0xffffffff81179e70 <d_free>
0xffffffff81179f3b <d_kill+107>:        mov    %r12,%rax
0xffffffff81179f3e <d_kill+110>:        mov    (%rsp),%rbx
0xffffffff81179f42 <d_kill+114>:        mov    0x8(%rsp),%r12
0xffffffff81179f47 <d_kill+119>:        leaveq
0xffffffff81179f48 <d_kill+120>:        retq
End of assembler dump.


/**
 * d_kill - kill dentry and return parent
 * @dentry: dentry to kill
 *
 * The dentry must already be unhashed and removed from the LRU.
 *
 * If this is the root of the dentry tree, return NULL.
 */
static struct dentry *d_kill(struct dentry *dentry)
    __releases(dentry->d_lock)
    __releases(dcache_lock)
{
    struct dentry *parent;

    list_del(&dentry->d_u.d_child);
    dentry_stat.nr_dentry--;    /* For d_free, below */
    /*drops the locks, at that point nobody can reach this dentry */
    dentry_iput(dentry);
    if (IS_ROOT(dentry))
        parent = NULL;
    else
        parent = dentry->d_parent;
    d_free(dentry);
    return parent;
}

crash> struct dentry
struct dentry {
    atomic_t d_count;
    unsigned int d_flags;
    spinlock_t d_lock;
    int d_mounted;
    struct inode *d_inode;
    struct hlist_node d_hash;
    struct dentry *d_parent;
    struct qstr d_name;
    struct list_head d_lru;
    union {
        struct list_head d_child;
        struct rcu_head d_rcu;
    } d_u;
    struct list_head d_subdirs;
    struct list_head d_alias;
    long unsigned int d_time;
    const struct dentry_operations *d_op;
    struct super_block *d_sb;
    void *d_fsdata;
    unsigned char d_iname[32];
}
SIZE: 192

crash> struct dentry ffff8805d305a180
struct dentry {
  d_count = {
    counter = 0
  },
  d_flags = 24,
  d_lock = {
    raw_lock = {
      slock = 458758
    }
  },
  d_mounted = 0,
  d_inode = 0x0,
  d_hash = {
    next = 0x0,
    pprev = 0xdead000000200200
  },
  d_parent = 0xffff88053dd10180,
  d_name = {
    hash = 3302437006,
    len = 24,
    name = 0xffff8805d305a220 "[0x2000abc6b:0x10bb:0x0]"
  },
  d_lru = {
    next = 0xffff8805d305a1c0,
    prev = 0xffff8805d305a1c0
  },
  d_u = {
    d_child = {
      next = 0xffff8803d633e710,
      prev = 0xffffffff8117a770
    },
    d_rcu = {
      next = 0xffff8803d633e710,
      func = 0xffffffff8117a770 <d_callback>
    }
  },
  d_subdirs = {
    next = 0xffff8805d305a1e0,
    prev = 0xffff8805d305a1e0
  },
  d_alias = {
    next = 0xffff8805d305a1f0,
    prev = 0xffff8805d305a1f0
  },
  d_time = 0,
  d_op = 0xffffffffa0a64140,
  d_sb = 0xffff88033cd03000,
  d_fsdata = 0xdeadbeef,
  d_iname = "[0x2000abc6b:0x10bb:0x0]\000\000\000\000\000\000\000"
}

crash> struct dentry 0xffff88053dd10180
struct dentry {
  d_count = {
    counter = 600395
  },
  d_flags = 16,
  d_lock = {
    raw_lock = {
      slock = 1896182021
    }
  },
  d_mounted = 1515870810,
  d_inode = 0xffff8804d9652b38,
  d_hash = {
    next = 0x0,
    pprev = 0xdead000000200200
  },
  d_parent = 0xffff88053dd100c0,
  d_name = {
    hash = 2401850,
    len = 3,
    name = 0xffff88053dd10220 "fid"
  },
  d_lru = {
    next = 0xffff88053dd101c0,
    prev = 0xffff88053dd101c0
  },
  d_u = {
    d_child = {
      next = 0xffff88053dd10120,
      prev = 0xffff88053dd10120
    },
    d_rcu = {
      next = 0xffff88053dd10120,
      func = 0xffff88053dd10120
    }
  },
  d_subdirs = {
    next = 0xffff880356ad4c10,
    prev = 0xffff880600534190
  },
  d_alias = {
    next = 0xffff8804d9652b68,
    prev = 0xffff8804d9652b68
  },
  d_time = 8083226193662651699,
  d_op = 0xffffffffa0a64140,
  d_sb = 0xffff88033cd03000,
  d_fsdata = 0xffff8804c20006c0,
  d_iname = "fid\000nity_hint\000x2af3:0x0]\000\000ZZZZZZ"
}

crash> disas dput
Dump of assembler code for function dput:
0xffffffff8117b8d0 <dput+0>:    push   %rbp
0xffffffff8117b8d1 <dput+1>:    mov    %rsp,%rbp
0xffffffff8117b8d4 <dput+4>:    push   %r12
0xffffffff8117b8d6 <dput+6>:    push   %rbx
0xffffffff8117b8d7 <dput+7>:    nopl   0x0(%rax,%rax,1)
0xffffffff8117b8dc <dput+12>:   test   %rdi,%rdi
0xffffffff8117b8df <dput+15>:   mov    %rdi,%rbx
0xffffffff8117b8e2 <dput+18>:   jne    0xffffffff8117b97d <dput+173>
0xffffffff8117b8e8 <dput+24>:   jmpq   0xffffffff8117ba00 <dput+304>
0xffffffff8117b8ed <dput+29>:   nopl   (%rax)
0xffffffff8117b8f0 <dput+32>:   mov    0x4(%rbx),%eax
0xffffffff8117b8f3 <dput+35>:   test   $0x10,%al
0xffffffff8117b8f5 <dput+37>:   jne    0xffffffff8117b920 <dput+80>
0xffffffff8117b8f7 <dput+39>:   or     $0x10,%eax
0xffffffff8117b8fa <dput+42>:   mov    0x20(%rbx),%rdx
0xffffffff8117b8fe <dput+46>:   mov    %eax,0x4(%rbx)
0xffffffff8117b901 <dput+49>:   mov    0x18(%rbx),%rax
0xffffffff8117b905 <dput+53>:   test   %rax,%rax
0xffffffff8117b908 <dput+56>:   mov    %rax,(%rdx)
0xffffffff8117b90b <dput+59>:   je     0xffffffff8117b911 <dput+65>
0xffffffff8117b90d <dput+61>:   mov    %rdx,0x8(%rax)
0xffffffff8117b911 <dput+65>:   mov    $0xdead000000200200,%r8
0xffffffff8117b91b <dput+75>:   mov    %r8,0x20(%rbx)
0xffffffff8117b91f <dput+79>:   nop
0xffffffff8117b920 <dput+80>:   mov    0x40(%rbx),%rax
0xffffffff8117b924 <dput+84>:   lea    0x40(%rbx),%rdx
0xffffffff8117b928 <dput+88>:   cmp    %rdx,%rax
0xffffffff8117b92b <dput+91>:   je     0xffffffff8117b969 <dput+153>
0xffffffff8117b92d <dput+93>:   mov    0x48(%rbx),%rdx
0xffffffff8117b931 <dput+97>:   mov    $0xdead000000100100,%rdi
0xffffffff8117b93b <dput+107>:  mov    $0xdead000000200200,%rsi
0xffffffff8117b945 <dput+117>:  mov    %rdx,0x8(%rax)
0xffffffff8117b949 <dput+121>:  mov    %rax,(%rdx)                <---- ICI le pointeur est OK
0xffffffff8117b94c <dput+124>:  mov    0x90(%rbx),%rax
0xffffffff8117b953 <dput+131>:  mov    %rdi,0x40(%rbx)
0xffffffff8117b957 <dput+135>:  mov    %rsi,0x48(%rbx)
0xffffffff8117b95b <dput+139>:  subl   $0x1,0x108(%rax)
0xffffffff8117b962 <dput+146>:  subl   $0x1,0x8b82fb(%rip)        # 0xffffffff81a33c64
0xffffffff8117b969 <dput+153>:  mov    %rbx,%rdi
0xffffffff8117b96c <dput+156>:  callq  0xffffffff81179ed0 <d_kill>              <-----    ICI on appelle d_kill()
0xffffffff8117b971 <dput+161>:  test   %rax,%rax
0xffffffff8117b974 <dput+164>:  mov    %rax,%rbx
0xffffffff8117b977 <dput+167>:  je     0xffffffff8117ba00 <dput+304>
0xffffffff8117b97d <dput+173>:  mov    (%rbx),%eax
0xffffffff8117b97f <dput+175>:  cmp    $0x1,%eax
0xffffffff8117b982 <dput+178>:  je     0xffffffff8117ba08 <dput+312>
0xffffffff8117b988 <dput+184>:  mov    $0xffffffff81a052c0,%rsi
0xffffffff8117b98f <dput+191>:  mov    %rbx,%rdi
0xffffffff8117b992 <dput+194>:  callq  0xffffffff81255890 <_atomic_dec_and_lock>
0xffffffff8117b997 <dput+199>:  test   %eax,%eax
0xffffffff8117b999 <dput+201>:  je     0xffffffff8117ba00 <dput+304>
0xffffffff8117b99b <dput+203>:  lea    0x8(%rbx),%r12
0xffffffff8117b99f <dput+207>:  mov    %r12,%rdi
0xffffffff8117b9a2 <dput+210>:  callq  0xffffffff814875a0 <_spin_lock>
0xffffffff8117b9a7 <dput+215>:  mov    (%rbx),%eax
0xffffffff8117b9a9 <dput+217>:  test   %eax,%eax
0xffffffff8117b9ab <dput+219>:  jne    0xffffffff8117b9e4 <dput+276>
0xffffffff8117b9ad <dput+221>:  mov    0x88(%rbx),%rax
0xffffffff8117b9b4 <dput+228>:  test   %rax,%rax
0xffffffff8117b9b7 <dput+231>:  je     0xffffffff8117b9cf <dput+255>
0xffffffff8117b9b9 <dput+233>:  mov    0x18(%rax),%rax
0xffffffff8117b9bd <dput+237>:  test   %rax,%rax
0xffffffff8117b9c0 <dput+240>:  je     0xffffffff8117b9cf <dput+255>
0xffffffff8117b9c2 <dput+242>:  mov    %rbx,%rdi
0xffffffff8117b9c5 <dput+245>:  callq  *%rax
0xffffffff8117b9c7 <dput+247>:  test   %eax,%eax
0xffffffff8117b9c9 <dput+249>:  jne    0xffffffff8117b8f0 <dput+32>
0xffffffff8117b9cf <dput+255>:  mov    0x4(%rbx),%eax
0xffffffff8117b9d2 <dput+258>:  test   $0x10,%al
0xffffffff8117b9d4 <dput+260>:  jne    0xffffffff8117b920 <dput+80>
0xffffffff8117b9da <dput+266>:  lea    0x40(%rbx),%rdx
0xffffffff8117b9de <dput+270>:  cmp    0x40(%rbx),%rdx
0xffffffff8117b9e2 <dput+274>:  je     0xffffffff8117ba12 <dput+322>
0xffffffff8117b9e4 <dput+276>:  mov    %r12,%rax
0xffffffff8117b9e7 <dput+279>:  incw   (%rax)
0xffffffff8117b9ea <dput+282>:  xchg   %ax,%ax
0xffffffff8117b9ed <dput+285>:  mov    $0xffffffff81a052c0,%rax
0xffffffff8117b9f4 <dput+292>:  incw   (%rax)
0xffffffff8117b9f7 <dput+295>:  xchg   %ax,%ax
0xffffffff8117b9fa <dput+298>:  nopw   0x0(%rax,%rax,1)
0xffffffff8117ba00 <dput+304>:  pop    %rbx
0xffffffff8117ba01 <dput+305>:  pop    %r12
0xffffffff8117ba03 <dput+307>:  leaveq
0xffffffff8117ba04 <dput+308>:  retq
0xffffffff8117ba05 <dput+309>:  nopl   (%rax)
0xffffffff8117ba08 <dput+312>:  callq  0xffffffff814855e0 <_cond_resched>
0xffffffff8117ba0d <dput+317>:  jmpq   0xffffffff8117b988 <dput+184>
0xffffffff8117ba12 <dput+322>:  or     $0x8,%eax
0xffffffff8117ba15 <dput+325>:  mov    %eax,0x4(%rbx)
0xffffffff8117ba18 <dput+328>:  mov    0x90(%rbx),%rax
0xffffffff8117ba1f <dput+335>:  mov    0xf8(%rax),%rcx
0xffffffff8117ba26 <dput+342>:  mov    %rdx,0x8(%rcx)
0xffffffff8117ba2a <dput+346>:  mov    %rcx,0x40(%rbx)
0xffffffff8117ba2e <dput+350>:  lea    0xf8(%rax),%rcx
0xffffffff8117ba35 <dput+357>:  mov    %rcx,0x48(%rbx)
0xffffffff8117ba39 <dput+361>:  mov    %rdx,0xf8(%rax)
0xffffffff8117ba40 <dput+368>:  mov    0x90(%rbx),%rax
0xffffffff8117ba47 <dput+375>:  addl   $0x1,0x108(%rax)
0xffffffff8117ba4e <dput+382>:  addl   $0x1,0x8b820f(%rip)        # 0xffffffff81a33c64
0xffffffff8117ba55 <dput+389>:  jmp    0xffffffff8117b9e4 <dput+276>
End of assembler dump.

 #5 [ffff8802e12e7c70] general_protection at ffffffff81487a25
    [exception RIP: d_kill+53]
    RIP: ffffffff81179f05  RSP: ffff8802e12e7d20  RFLAGS: 00010282
    RAX: dead000000200200  RBX: ffff8805d305a180  RCX: ffff8805d305a1c0
    RDX: dead000000100100  RSI: dead000000200200  RDI: dead000000100100
    RBP: ffff8802e12e7d30   R8: 00000000deadbeef   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 5a5a5a5a5a5a5a5a  R12: ffff8805d305a188
    R13: ffffffffa0a848a0  R14: ffff88053dd101e0  R15: ffff8804d9652af8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8802e12e7c78: ffff8804d9652af8 ffff88053dd101e0
    ffff8802e12e7c88: ffffffffa0a848a0 ffff8805d305a188
    ffff8802e12e7c98: ffff8802e12e7d30 ffff8805d305a180
    ffff8802e12e7ca8: 5a5a5a5a5a5a5a5a 5a5a5a5a5a5a5a5a
    ffff8802e12e7cb8: 5a5a5a5a5a5a5a5a 00000000deadbeef
    ffff8802e12e7cc8: dead000000200200 ffff8805d305a1c0
    ffff8802e12e7cd8: dead000000100100 dead000000200200
    ffff8802e12e7ce8: dead000000100100 ffffffffffffffff
    ffff8802e12e7cf8: ffffffff81179f05 0000000000000010
    ffff8802e12e7d08: 0000000000010282 ffff8802e12e7d20
    ffff8802e12e7d18: 0000000000000018 ffff8805d305a180
    ffff8802e12e7d28: ffff8805d305a188 ffff8802e12e7d50
    ffff8802e12e7d38: ffffffff8117b971
 #6 [ffff8802e12e7d38] dput at ffffffff8117b971
    ffff8802e12e7d40: ffff8805d305a180 ffffffff81a052c0
    ffff8802e12e7d50: ffff8802e12e7d90 ffffffffa09eb626
 #7 [ffff8802e12e7d58] ll_drop_dentry at ffffffffa09eb626 [lustre]
    ffff8802e12e7d60: ffffffff81a052c0 ffff880044892bc0
    ffff8802e12e7d70: ffffffffffffff02 ffff8804d9652b38
    ffff8802e12e7d80: dead0000001000b0 0000000000000003
    ffff8802e12e7d90: ffff8802e12e7e00 ffffffffa0a2c702
 #8 [ffff8802e12e7d98] ll_md_blocking_ast at ffffffffa0a2c702 [lustre]


crash> x/a 0xffff8805d305a180 + 0x50
0xffff8805d305a1d0:     0xffff8803d633e710
crash> x/a 0xffff8805d305a180 + 0x58
0xffff8805d305a1d8:     0xffffffff8117a770 <d_callback>



0xffffffff81179edc <d_kill+12>: mov    %r12,0x8(%rsp)
0xffffffff81179ee1 <d_kill+17>: nopl   0x0(%rax,%rax,1)
0xffffffff81179ee6 <d_kill+22>: mov    0x50(%rdi),%rdx     <--- rdi = 0xffff8805d305a180=rbx  struct dentry.d_u.d_child.next
0xffffffff81179eea <d_kill+26>: mov    0x58(%rdi),%rax     <----rdi = 0xffff8805d305a180=rbx  struct dentry.d_u.d_child.prev
0xffffffff81179eee <d_kill+30>: mov    $0xdead000000200200,%rsi
0xffffffff81179ef8 <d_kill+40>: mov    %rdi,%rbx
0xffffffff81179efb <d_kill+43>: mov    $0xdead000000100100,%rdi
0xffffffff81179f05 <d_kill+53>: mov    %rax,0x8(%rdx)      <--- ICI RDX: dead000000100100 n'est pas un pointeur   rax=dead000000200200
0xffffffff81179f09 <d_kill+57>: mov    %rdx,(%rax)
0xffffffff81179f0c <d_kill+60>: mov    %rdi,0x50(%rbx)


entre 0xffffffff81179ee6 et 0xffffffff81179f05 la valeur de 0xffff8805d305a180 + 0x50 a changer ....
et est passer de 0xdead000000100100 a 0xffff8803d633e710



static inline void __list_del(struct list_head * prev, struct list_head * next)
{
    next->prev = prev;
    prev->next = next;
}

/**
 * list_del - deletes entry from list.
 * @entry: the element to delete from the list.
 * Note: list_empty() on entry does not return true after this, the entry is
 * in an undefined state.
 */
#ifndef CONFIG_DEBUG_LIST
static inline void list_del(struct list_head *entry)
{
    __list_del(entry->prev, entry->next);
    entry->next = LIST_POISON1;
    entry->prev = LIST_POISON2;
}
#else
extern void list_del(struct list_head *entry);
#endif


/*
 * These are non-NULL pointers that will result in page faults
 * under normal circumstances, used to verify that nobody uses
 * non-initialized list entries.
 */
#define LIST_POISON1  ((void *) 0x00100100 + POISON_POINTER_DELTA)
#define LIST_POISON2  ((void *) 0x00200200 + POISON_POINTER_DELTA)

[root@germain134 WW.WW.0.1-2012-12-03-19:01:28]# /cea/local/bull/bin/read_lustre_debug_trace_from_crash.sh
Traitement CPU:0 0xffff88033d06c010 Mon Feb 11 17:33:25 CET 2013
Traitement CPU:1 0xffff88033d06c090 Mon Feb 11 17:34:02 CET 2013
Traitement CPU:2 0xffff88033d06c110 Mon Feb 11 17:34:40 CET 2013
Traitement CPU:3 0xffff88033d06c190 Mon Feb 11 17:36:17 CET 2013
Traitement CPU:4 0xffff88033d06c210 Mon Feb 11 17:37:15 CET 2013
Traitement CPU:5 0xffff88033d06c290 Mon Feb 11 17:38:35 CET 2013
Traitement CPU:6 0xffff88033d06c310 Mon Feb 11 17:39:34 CET 2013
Traitement CPU:7 0xffff88033d06c390 Mon Feb 11 17:40:39 CET 2013

dentry trace could be useful ....

PID: 10428  TASK: ffff88033d2c1850  CPU: 0   COMMAND: "umount"
 #0 [ffff880028207e90] crash_nmi_callback at ffffffff81020b56
 #1 [ffff880028207ea0] notifier_call_chain at ffffffff8148a755
 #2 [ffff880028207ee0] atomic_notifier_call_chain at ffffffff8148a7ba
 #3 [ffff880028207ef0] notify_die at ffffffff810817ce
 #4 [ffff880028207f20] do_nmi at ffffffff814883d3
 #5 [ffff880028207f50] nmi at ffffffff81487ce0
    [exception RIP: ldlm_res_hop_fid_hash+25]
    RIP: ffffffffa05f43b9  RSP: ffff880120b2bb98  RFLAGS: 00000292
    RAX: ffffffffa06acda0  RBX: ffff8805f7cbc300  RCX: 0000000000000010
    RDX: 000000000000ffff  RSI: ffff880120b2bcc8  RDI: ffff8805f7cbc300
    RBP: ffff880120b2bbd8   R8: ffff880120b2bc48   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 00000002000abc55  R12: 0000000000000010
    R13: ffff8805f8216bc0  R14: ffff880120b2bc48  R15: 0000000000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
--- <NMI exception stack> ---
 #6 [ffff880120b2bb98] ldlm_res_hop_fid_hash at ffffffffa05f43b9 [ptlrpc]
 #7 [ffff880120b2bbe0] cfs_hash_bd_from_key at ffffffffa03db192 [libcfs]
 #8 [ffff880120b2bc10] cfs_hash_bd_get at ffffffffa03db8a5 [libcfs]
 #9 [ffff880120b2bc20] ldlm_resource_get at ffffffffa05f7099 [ptlrpc]
#10 [ffff880120b2bc90] ldlm_resource_iterate at ffffffffa0609b1d [ptlrpc]
#11 [ffff880120b2bcc0] mdc_find_cbdata at ffffffffa08a63cc [mdc]
#12 [ffff880120b2bd20] lmv_find_cbdata at ffffffffa0b17501 [lmv]
#13 [ffff880120b2bd80] find_cbdata at ffffffffa09ecdd2 [lustre]
#14 [ffff880120b2bdb0] ll_d_iput at ffffffffa09ed8be [lustre]
#15 [ffff880120b2bdd0] shrink_dcache_for_umount_subtree at ffffffff8117b36d
#16 [ffff880120b2be20] shrink_dcache_for_umount at ffffffff8117b4f6
#17 [ffff880120b2be40] generic_shutdown_super at ffffffff811653ef
#18 [ffff880120b2be60] kill_anon_super at ffffffff81165546
#19 [ffff880120b2be80] lustre_kill_super at ffffffffa04c699a [obdclass]
#20 [ffff880120b2bea0] deactivate_super at ffffffff811664e0
#21 [ffff880120b2bec0] mntput_no_expire at ffffffff811826bf
#22 [ffff880120b2bef0] sys_umount at ffffffff81183188
#23 [ffff880120b2bf80] system_call_fastpath at ffffffff810030f2
    RIP: 00007fd4f64ccd67  RSP: 00007fff034b8a38  RFLAGS: 00010202
    RAX: 00000000000000a6  RBX: ffffffff810030f2  RCX: 0000000000000010
    RDX: 0000000000000000  RSI: 0000000000000000  RDI: 00007fd4f82f8f80
    RBP: 00007fd4f82f8f40   R8: 00007fd4f82f8fa0   R9: 0000000000000000
    R10: 00007fff034b8860  R11: 0000000000000246  R12: 0000000000000000
    R13: 0000000000000000  R14: 0000000000000000  R15: 00007fd4f82f8fe0
    ORIG_RAX: 00000000000000a6  CS: 0033  SS: 002b

PID: 13878  TASK: ffff88063c92f080  CPU: 2   COMMAND: "java"
 #0 [ffff880028247e90] crash_nmi_callback at ffffffff81020b56
 #1 [ffff880028247ea0] notifier_call_chain at ffffffff8148a755
 #2 [ffff880028247ee0] atomic_notifier_call_chain at ffffffff8148a7ba
 #3 [ffff880028247ef0] notify_die at ffffffff810817ce
 #4 [ffff880028247f20] do_nmi at ffffffff814883d3
 #5 [ffff880028247f50] nmi at ffffffff81487ce0
    [exception RIP: _spin_lock+30]
    RIP: ffffffff814875be  RSP: ffff88063d51bc18  RFLAGS: 00000297
    RAX: 0000000000004645  RBX: ffff880293e7d440  RCX: 0000000000000000
    RDX: 0000000000004644  RSI: ffff88025651801a  RDI: ffffffff81a052c0
    RBP: ffff88063d51bc18   R8: 0000000000000001   R9: 0000000000000004
    R10: 0000000000000000  R11: 0000000000000246  R12: ffff8804b0ede200
    R13: ffff88063d51bd08  R14: ffff880293e7d4e0  R15: ffff88059b059bc0
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
--- <NMI exception stack> ---
 #6 [ffff88063d51bc18] _spin_lock at ffffffff814875be
 #7 [ffff88063d51bc20] d_alloc at ffffffff8117bca4
 #8 [ffff88063d51bc50] do_lookup at ffffffff811716fd
 #9 [ffff88063d51bca0] __link_path_walk at ffffffff81171a9d
#10 [ffff88063d51bd60] path_walk at ffffffff81172b4a
#11 [ffff88063d51bda0] do_path_lookup at ffffffff81172d1b
#12 [ffff88063d51bdd0] user_path_at at ffffffff81173987
#13 [ffff88063d51bea0] vfs_fstatat at ffffffff81168536
#14 [ffff88063d51bee0] vfs_stat at ffffffff8116869b
#15 [ffff88063d51bef0] sys_newstat at ffffffff811686c4
#16 [ffff88063d51bf80] system_call_fastpath at ffffffff810030f2
    RIP: 00007ff57f0ab2e5  RSP: 00007ff3c488a638  RFLAGS: 00010246
    RAX: 0000000000000004  RBX: ffffffff810030f2  RCX: 0000000000000000
    RDX: 00007ff3c4889e60  RSI: 00007ff3c4889e60  RDI: 00007ff3c4889ef0
    RBP: 00007ff3c4889f70   R8: 0000000000000000   R9: 0000000000000000
    R10: 0000000000000022  R11: 0000000000000246  R12: 0000000000000009
    R13: 00007ff380009370  R14: 0000000000000000  R15: 00007ff380009370
    ORIG_RAX: 0000000000000004  CS: 0033  SS: 002b

PID: 26203  TASK: ffff88033d5dc040  CPU: 7   COMMAND: "ldlm_bl_44"
 #0 [ffff8802e12e7ab0] machine_kexec at ffffffff8102895b
 #1 [ffff8802e12e7b10] crash_kexec at ffffffff810a4622
 #2 [ffff8802e12e7be0] oops_end at ffffffff814886c0
 #3 [ffff8802e12e7c10] die at ffffffff8100714b
 #4 [ffff8802e12e7c40] do_general_protection at ffffffff81488252
 #5 [ffff8802e12e7c70] general_protection at ffffffff81487a25
    [exception RIP: d_kill+53]
    RIP: ffffffff81179f05  RSP: ffff8802e12e7d20  RFLAGS: 00010282
    RAX: dead000000200200  RBX: ffff8805d305a180  RCX: ffff8805d305a1c0
    RDX: dead000000100100  RSI: dead000000200200  RDI: dead000000100100
    RBP: ffff8802e12e7d30   R8: 00000000deadbeef   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 5a5a5a5a5a5a5a5a  R12: ffff8805d305a188
    R13: ffffffffa0a848a0  R14: ffff88053dd101e0  R15: ffff8804d9652af8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #6 [ffff8802e12e7d38] dput at ffffffff8117b971
 #7 [ffff8802e12e7d58] ll_drop_dentry at ffffffffa09eb626 [lustre]
 #8 [ffff8802e12e7d98] ll_md_blocking_ast at ffffffffa0a2c702 [lustre]
 #9 [ffff8802e12e7e08] ldlm_cancel_callback at ffffffffa05f01f0 [ptlrpc]
#10 [ffff8802e12e7e28] ldlm_cli_cancel_local at ffffffffa060a20b [ptlrpc]
#11 [ffff8802e12e7e58] ldlm_cli_cancel_list_local at ffffffffa060cafd [ptlrpc]
#12 [ffff8802e12e7eb8] ldlm_bl_thread_main at ffffffffa06114f0 [ptlrpc]
#13 [ffff8802e12e7f48] kernel_thread at ffffffff8100412a

PID: 28936  TASK: ffff88063be6d810  CPU: 6   COMMAND: "lctl"
 #0 [ffff8801545479e8] schedule at ffffffff81484c15
 #1 [ffff880154547ab0] schedule_timeout at ffffffff81485ab5
 #2 [ffff880154547b60] wait_for_common at ffffffff814856f7
 #3 [ffff880154547bf0] wait_for_completion at ffffffff8148585d
 #4 [ffff880154547c00] __ldlm_bl_to_thread at ffffffffa061086d [ptlrpc]
 #5 [ffff880154547c30] ldlm_bl_to_thread at ffffffffa061449d [ptlrpc]
 #6 [ffff880154547d30] ldlm_bl_to_thread_list at ffffffffa0614690 [ptlrpc]
 #7 [ffff880154547d40] ldlm_cancel_lru at ffffffffa060d840 [ptlrpc]
 #8 [ffff880154547d90] lprocfs_wr_lru_size at ffffffffa05f784f [ptlrpc]
 #9 [ffff880154547e60] lprocfs_fops_write at ffffffffa04ad26b [obdclass]
#10 [ffff880154547ea0] proc_reg_write at ffffffff811c843e
#11 [ffff880154547ef0] vfs_write at ffffffff811631d8
#12 [ffff880154547f30] sys_write at ffffffff81163be1
#13 [ffff880154547f80] system_call_fastpath at ffffffff810030f2
    RIP: 00007f8a6fc39a10  RSP: 00007fffbafffdb8  RFLAGS: 00010206
    RAX: 0000000000000001  RBX: ffffffff810030f2  RCX: 00007f8a6fc397c0
    RDX: 0000000000000005  RSI: 00007fffbb002da2  RDI: 0000000000000003
    RBP: 0000000000000314   R8: 00007fffbb002da2   R9: 0000000000000000
    R10: 00000000fffffffb  R11: 0000000000000246  R12: 00007fffbb002da2
    R13: 0000000000000000  R14: 0000000000000003  R15: 0000000000000002
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b

crash> ps -l
[1063346611036521]  PID: 23     TASK: ffff88033e5c5850  CPU: 6   COMMAND: "ksoftirqd/6"
[1063346580359790]  PID: 5789   TASK: ffff88033d66d0c0  CPU: 5   COMMAND: "robinhood"
[1063346578164264]  PID: 13857  TASK: ffff88033d090790  CPU: 5   COMMAND: "java"
[1063346538406674]  PID: 13817  TASK: ffff88063e60f810  CPU: 5   COMMAND: "java"
[1063346302681733]  PID: 26545  TASK: ffff8802df5d2100  CPU: 5   COMMAND: "ldlm_poold"
[1063346272323389]  PID: 24252  TASK: ffff88063c9967d0  CPU: 3   COMMAND: "router_checker"
[1063346266575091]  PID: 13577  TASK: ffff88063cbf60c0  CPU: 1   COMMAND: "nrpe"
[1063346122811200]  PID: 12593  TASK: ffff88033c3ad0c0  CPU: 4   COMMAND: "mlx4_sense"
[1063346113262074]  PID: 25411  TASK: ffff88063d664790  CPU: 6   COMMAND: "ptlrpcd_7"
[1063346113252422]  PID: 25409  TASK: ffff88063c621080  CPU: 6   COMMAND: "ptlrpcd_5"
[1063346112658544]  PID: 25403  TASK: ffff88063d665810  CPU: 3   COMMAND: "ptlrpcd_0"
[1063346112656454]  PID: 25406  TASK: ffff88063d7e17d0  CPU: 3   COMMAND: "ptlrpcd_2"
[1063346112654119]  PID: 25405  TASK: ffff88063cd407d0  CPU: 3   COMMAND: "ptlrpcd_1"
[1063346112651766]  PID: 25408  TASK: ffff88063cd40080  CPU: 3   COMMAND: "ptlrpcd_4"
[1063346112648800]  PID: 25410  TASK: ffff88063c996080  CPU: 3   COMMAND: "ptlrpcd_6"
[1063346112631517]  PID: 25407  TASK: ffff88060f28a810  CPU: 3   COMMAND: "ptlrpcd_3"

static int do_lookup(struct nameidata *nd, struct qstr *name,
             struct path *path)
{
    struct vfsmount *mnt = nd->path.mnt;
    struct dentry *dentry = __d_lookup(nd->path.dentry, name);
    struct dentry *parent;
    struct inode *dir;
    int err;

    if (!dentry)
        goto need_lookup;
found:
    if (dentry->d_op && dentry->d_op->d_revalidate)
        goto need_revalidate;
done:
    path->mnt = mnt;
    path->dentry = dentry;
    err = follow_managed(path, nd->flags);
    if (unlikely(err < 0))
        path_put_conditional(path, nd);
    return err;

need_lookup:
    parent = nd->path.dentry;
    dir = parent->d_inode;

    mutex_lock(&dir->i_mutex);
    /*
     * First re-do the cached lookup just in case it was created
     * while we waited for the directory semaphore..
     *
     * FIXME! This could use version numbering or similar to
     * avoid unnecessary cache lookups.
     *
     * The "dcache_lock" is purely to protect the RCU list walker
     * from concurrent renames at this point (we mustn't get false
     * negatives from the RCU list walk here, unlike the optimistic
     * fast walk).
     *
     * so doing d_lookup() (with seqlock), instead of lockfree __d_lookup
     */
    dentry = d_lookup(parent, name);
    if (!dentry) {
        struct dentry *new;

        /* Don't create child dentry for a dead directory. */
        dentry = ERR_PTR(-ENOENT);
        if (IS_DEADDIR(dir))
            goto out_unlock;

        new = d_alloc(parent, name);
        dentry = ERR_PTR(-ENOMEM);
        if (new) {
            dentry = dir->i_op->lookup(dir, new, nd);
            if (dentry)
                dput(new);
            else
                dentry = new;
        }
out_unlock:
        mutex_unlock(&dir->i_mutex);
        if (IS_ERR(dentry))
            goto fail;
        goto done;
    }

    /*
     * Uhhuh! Nasty case: the cache was re-populated while
     * we waited on the semaphore. Need to revalidate.
     */
    mutex_unlock(&dir->i_mutex);
    goto found;

need_revalidate:
    dentry = do_revalidate(dentry, nd);
    if (!dentry)
        goto need_lookup;
    if (IS_ERR(dentry))
        goto fail;
    goto done;

fail:
    return PTR_ERR(dentry);
}


Race condition on d_entry structure manipulation 

crash> bt 13878
PID: 13878  TASK: ffff88063c92f080  CPU: 2   COMMAND: "java"
 #0 [ffff880028247e90] crash_nmi_callback at ffffffff81020b56
 #1 [ffff880028247ea0] notifier_call_chain at ffffffff8148a755
 #2 [ffff880028247ee0] atomic_notifier_call_chain at ffffffff8148a7ba
 #3 [ffff880028247ef0] notify_die at ffffffff810817ce
 #4 [ffff880028247f20] do_nmi at ffffffff814883d3
 #5 [ffff880028247f50] nmi at ffffffff81487ce0
    [exception RIP: _spin_lock+30]
    RIP: ffffffff814875be  RSP: ffff88063d51bc18  RFLAGS: 00000297
    RAX: 0000000000004645  RBX: ffff880293e7d440  RCX: 0000000000000000
    RDX: 0000000000004644  RSI: ffff88025651801a  RDI: ffffffff81a052c0
    RBP: ffff88063d51bc18   R8: 0000000000000001   R9: 0000000000000004
    R10: 0000000000000000  R11: 0000000000000246  R12: ffff8804b0ede200
    R13: ffff88063d51bd08  R14: ffff880293e7d4e0  R15: ffff88059b059bc0
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
--- <NMI exception stack> ---
 #6 [ffff88063d51bc18] _spin_lock at ffffffff814875be
 #7 [ffff88063d51bc20] d_alloc at ffffffff8117bca4
 #8 [ffff88063d51bc50] do_lookup at ffffffff811716fd
 #9 [ffff88063d51bca0] __link_path_walk at ffffffff81171a9d
#10 [ffff88063d51bd60] path_walk at ffffffff81172b4a
#11 [ffff88063d51bda0] do_path_lookup at ffffffff81172d1b
#12 [ffff88063d51bdd0] user_path_at at ffffffff81173987
#13 [ffff88063d51bea0] vfs_fstatat at ffffffff81168536
#14 [ffff88063d51bee0] vfs_stat at ffffffff8116869b
#15 [ffff88063d51bef0] sys_newstat at ffffffff811686c4
#16 [ffff88063d51bf80] system_call_fastpath at ffffffff810030f2
    RIP: 00007ff57f0ab2e5  RSP: 00007ff3c488a638  RFLAGS: 00010246
    RAX: 0000000000000004  RBX: ffffffff810030f2  RCX: 0000000000000000
    RDX: 00007ff3c4889e60  RSI: 00007ff3c4889e60  RDI: 00007ff3c4889ef0
    RBP: 00007ff3c4889f70   R8: 0000000000000000   R9: 0000000000000000
    R10: 0000000000000022  R11: 0000000000000246  R12: 0000000000000009
    R13: 00007ff380009370  R14: 0000000000000000  R15: 00007ff380009370
    ORIG_RAX: 0000000000000004  CS: 0033  SS: 002b
crash> bt -f 13878
PID: 13878  TASK: ffff88063c92f080  CPU: 2   COMMAND: "java"
 #0 [ffff880028247e90] crash_nmi_callback at ffffffff81020b56
    ffff880028247e98: ffff880028247ed8 ffffffff8148a755
 #1 [ffff880028247ea0] notifier_call_chain at ffffffff8148a755
    ffff880028247ea8: 0000000000000000 ffff880028247f58
    ffff880028247eb8: 0000000000000000 0000000000000000
    ffff880028247ec8: ffff880293e7d4e0 ffff88059b059bc0
    ffff880028247ed8: ffff880028247ee8 ffffffff8148a7ba
 #2 [ffff880028247ee0] atomic_notifier_call_chain at ffffffff8148a7ba
    ffff880028247ee8: ffff880028247f18 ffffffff810817ce
 #3 [ffff880028247ef0] notify_die at ffffffff810817ce
    ffff880028247ef8: ffff880028247f58 ffffffff8175a4c2
    ffff880028247f08: 0000000000000000 0000000200000002
    ffff880028247f18: ffff880028247f48 ffffffff814883d3
 #4 [ffff880028247f20] do_nmi at ffffffff814883d3
    ffff880028247f28: 0000000000000000 0000000000000001
    ffff880028247f38: ffff8804b0ede200 ffff88063d51bd08
    ffff880028247f48: ffff88063d51bc18 ffffffff81487ce0
 #5 [ffff880028247f50] nmi at ffffffff81487ce0
    [exception RIP: _spin_lock+30]
    RIP: ffffffff814875be  RSP: ffff88063d51bc18  RFLAGS: 00000297
    RAX: 0000000000004645  RBX: ffff880293e7d440  RCX: 0000000000000000
    RDX: 0000000000004644  RSI: ffff88025651801a  RDI: ffffffff81a052c0
    RBP: ffff88063d51bc18   R8: 0000000000000001   R9: 0000000000000004
    R10: 0000000000000000  R11: 0000000000000246  R12: ffff8804b0ede200
    R13: ffff88063d51bd08  R14: ffff880293e7d4e0  R15: ffff88059b059bc0
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
--- <NMI exception stack> ---
 #6 [ffff88063d51bc18] _spin_lock at ffffffff814875be
    ffff88063d51bc20: ffffffff8117bca4
 #7 [ffff88063d51bc20] d_alloc at ffffffff8117bca4
    ffff88063d51bc28: ffff88063d51bdd8 ffff88063d51bd18
    ffff88063d51bc38: ffff88063d51bd08 ffff88033cf64cc0
    ffff88063d51bc48: ffff88063d51bc98 ffffffff811716fd
 #8 [ffff88063d51bc50] do_lookup at ffffffff811716fd
    ffff88063d51bc58: ffff88059b059b08 ffff8804b0ede200
    ffff88063d51bc68: ffff88063d51bc98 ffff88025651801b
    ffff88063d51bc78: ffff88063d51bdd8 ffff88059b059b08
    ffff88063d51bc88: ffff88063c92f080 ffff88063d51bd18
    ffff88063d51bc98: ffff88063d51bd58 ffffffff81171a9d
 #9 [ffff88063d51bca0] __link_path_walk at ffffffff81171a9d
    ffff88063d51bca8: ffff88063d51bd08 00000000fffffffe
    ffff88063d51bcb8: ffff88033e52f550 ffff88063c92f080
    ffff88063d51bcc8: ffff88063c92f080 ffff88063c92f080
    ffff88063d51bcd8: ffff88063c92f080 0000000000000000
    ffff88063d51bce8: ffff88063c92f080 0000000105b3f738
    ffff88063d51bcf8: 0000000000000000 ffff88063e45af50
    ffff88063d51bd08: 0000000511bf240d ffff880256518015
    ffff88063d51bd18: ffff88033cf64cc0 ffff8804b0ede200
    ffff88063d51bd28: 00000000fffffffe ffff88063d51bdd8
    ffff88063d51bd38: ffff880256518000 00000000ffffff9c
    ffff88063d51bd48: 0000000000000001 00007ff3c4889ef0
    ffff88063d51bd58: ffff88063d51bd98 ffffffff81172b4a
#10 [ffff88063d51bd60] path_walk at ffffffff81172b4a
    ffff88063d51bd68: ffff88063e45ae80 ffff88063e1adec0
    ffff88063d51bd78: 00007ff3c4889ef0 ffff88063d51bdd8
    ffff88063d51bd88: ffff880256518000 00000000ffffff9c
    ffff88063d51bd98: ffff88063d51bdc8 ffffffff81172d1b
#11 [ffff88063d51bda0] do_path_lookup at ffffffff81172d1b
    ffff88063d51bda8: ffff88063d51beb8 ffff88063d51beb8
    ffff88063d51bdb8: ffff880256518000 ffff88063d51beb8
    ffff88063d51bdc8: ffff88063d51be98 ffffffff81173987
#12 [ffff88063d51bdd0] user_path_at at ffffffff81173987
    ffff88063d51bdd8: ffff88033cf64cc0 ffff8804b0ede200
    ffff88063d51bde8: ffffffff81bcd810 ffff88063ced8708
    ffff88063d51bdf8: ffff88063e45ae80 ffff88063e1adec0
    ffff88063d51be08: 0000000100000005 ffff880000000000
    ffff88063d51be18: 000000000000000e ffffea0005b3f738
    ffff88063d51be28: ffff88063d51be78 ffffffff81138cb4
    ffff88063d51be38: ffff88063ced8708 ffff8801f91d43a0
    ffff88063d51be48: ffff88063ced8440 ffff88002824cac0
    ffff88063d51be58: ffff88063d51bec8 0000000000000282
    ffff88063d51be68: ffff88063ced8440 00007ff57f8c3000
    ffff88063d51be78: ffff88063d51beb8 ffff88063d51bef8
    ffff88063d51be88: 00007ff3c4889c80 00007ff380001340
    ffff88063d51be98: ffff88063d51bed8 ffffffff81168536
#13 [ffff88063d51bea0] vfs_fstatat at ffffffff81168536
    ffff88063d51bea8: ffff8801f91d43a0 ffff88063be6cd00
    ffff88063d51beb8: ffff8801f91d43a0 00007ff57f8c4000
    ffff88063d51bec8: ffff88063d51bef8 00007ff3c4889e60
    ffff88063d51bed8: ffff88063d51bee8 ffffffff8116869b
#14 [ffff88063d51bee0] vfs_stat at ffffffff8116869b
    ffff88063d51bee8: ffff88063d51bf78 ffffffff811686c4
#15 [ffff88063d51bef0] sys_newstat at ffffffff811686c4
    ffff88063d51bef8: 00007ff380001340 ffff8801f91d43a0
    ffff88063d51bf08: ffff88063be6cd18 ffff88063ced8448
    ffff88063d51bf18: 0000000000001000 ffff88063ced84a8
    ffff88063d51bf28: ffff88063ced8440 00007ff57f8c3000
    ffff88063d51bf38: 0000000000001000 00007ff3c4889ef0
    ffff88063d51bf48: ffff88063d51bf78 ffffffff8112e7d1
    ffff88063d51bf58: 00007ff380009370 0000000000000000
    ffff88063d51bf68: 00007ff380009370 0000000000000009
    ffff88063d51bf78: 00007ff3c4889f70 ffffffff810030f2
#16 [ffff88063d51bf80] system_call_fastpath at ffffffff810030f2
    RIP: 00007ff57f0ab2e5  RSP: 00007ff3c488a638  RFLAGS: 00010246
    RAX: 0000000000000004  RBX: ffffffff810030f2  RCX: 0000000000000000
    RDX: 00007ff3c4889e60  RSI: 00007ff3c4889e60  RDI: 00007ff3c4889ef0
    RBP: 00007ff3c4889f70   R8: 0000000000000000   R9: 0000000000000000
    R10: 0000000000000022  R11: 0000000000000246  R12: 0000000000000009
    R13: 00007ff380009370  R14: 0000000000000000  R15: 00007ff380009370
    ORIG_RAX: 0000000000000004  CS: 0033  SS: 002b

Dump of assembler code for function __link_path_walk:
0xffffffff81171890 <__link_path_walk+0>:        push   %rbp                
0xffffffff81171891 <__link_path_walk+1>:        mov    %rsp,%rbp
0xffffffff81171894 <__link_path_walk+4>:        push   %r15
0xffffffff81171896 <__link_path_walk+6>:        push   %r14
0xffffffff81171898 <__link_path_walk+8>:        push   %r13
0xffffffff8117189a <__link_path_walk+10>:       push   %r12
0xffffffff8117189c <__link_path_walk+12>:       push   %rbx
0xffffffff8117189d <__link_path_walk+13>:       sub    $0x88,%rsp

0xffffffff81171a79 <__link_path_walk+489>:      test   %rax,%rax
0xffffffff81171a7c <__link_path_walk+492>:      je     0xffffffff81171a8e <__link_path_walk+510>
0xffffffff81171a7e <__link_path_walk+494>:      lea    -0x50(%rbp),%rsi
0xffffffff81171a82 <__link_path_walk+498>:      callq  *%rax
0xffffffff81171a84 <__link_path_walk+500>:      test   %eax,%eax
0xffffffff81171a86 <__link_path_walk+502>:      mov    %eax,%ecx
0xffffffff81171a88 <__link_path_walk+504>:      js     0xffffffff81171947 <__link_path_walk+183>
0xffffffff81171a8e <__link_path_walk+510>:      lea    -0x50(%rbp),%rsi
0xffffffff81171a92 <__link_path_walk+514>:      mov    %r15,%rdx
0xffffffff81171a95 <__link_path_walk+517>:      mov    %r12,%rdi
0xffffffff81171a98 <__link_path_walk+520>:      callq  0xffffffff811715b0 <do_lookup>
0xffffffff81171a9d <__link_path_walk+525>:      test   %eax,%eax


Dump of assembler code for function do_lookup:
0xffffffff811715b0 <do_lookup+0>:       push   %rbp               -0x50(%rbp) --> rsi 2  0xffff88063d51bd58 - 0x50 = 0x0000000511bf240d   struct qstr *name
0xffffffff811715b1 <do_lookup+1>:       mov    %rsp,%rbp
0xffffffff811715b4 <do_lookup+4>:       sub    $0x40,%rsp
0xffffffff811715b8 <do_lookup+8>:       mov    %rbx,-0x28(%rbp)
0xffffffff811715bc <do_lookup+12>:      mov    %r12,-0x20(%rbp)   (rdi) 1    x/a 0xffff88063d51bc98 - 0x20 0xffff88063d51bc78:     0xffff88063d51bdd8  struct nameidata *nd
0xffffffff811715c0 <do_lookup+16>:      mov    %r13,-0x18(%rbp)
0xffffffff811715c4 <do_lookup+20>:      mov    %r14,-0x10(%rbp)
0xffffffff811715c8 <do_lookup+24>:      mov    %r15,-0x8(%rbp)    (rdx) 3    struct path *path   0xffff88063d51bc98 - 0x8 0xffff88063d51bc90:     0xffff88063d51bd18
0xffffffff811715cc <do_lookup+28>:      nopl   0x0(%rax,%rax,1)
0xffffffff811715d1 <do_lookup+33>:      mov    (%rdi),%r14
0xffffffff811715d4 <do_lookup+36>:      mov    %rdi,%rbx

crash> kmem 0xffff88063d51bdd8
    PID: 13878
COMMAND: "java"
   TASK: ffff88063c92f080  [THREAD_INFO: ffff88063d518000]
    CPU: 2
  STATE: TASK_RUNNING (ACTIVE)

      PAGE        PHYSICAL      MAPPING       INDEX CNT FLAGS
ffffea0015d69de8 63d51b000                0       11  0 c00000000000000



 #8 [ffff88063d51bc50] do_lookup at ffffffff811716fd
    ffff88063d51bc58: ffff88059b059b08 ffff8804b0ede200
    ffff88063d51bc68: ffff88063d51bc98 ffff88025651801b
    ffff88063d51bc78: ffff88063d51bdd8 ffff88059b059b08     r12/rdi   
    ffff88063d51bc88: ffff88063c92f080 ffff88063d51bd18
    ffff88063d51bc98: ffff88063d51bd58 ffffffff81171a9d
 #9 [ffff88063d51bca0] __link_path_walk at ffffffff81171a9d
    ffff88063d51bca8: ffff88063d51bd08 00000000fffffffe
    ffff88063d51bcb8: ffff88033e52f550 ffff88063c92f080
    ffff88063d51bcc8: ffff88063c92f080 ffff88063c92f080
    ffff88063d51bcd8: ffff88063c92f080 0000000000000000
    ffff88063d51bce8: ffff88063c92f080 0000000105b3f738
    ffff88063d51bcf8: 0000000000000000 ffff88063e45af50
    ffff88063d51bd08: 0000000511bf240d ffff880256518015
    ffff88063d51bd18: ffff88033cf64cc0 ffff8804b0ede200
    ffff88063d51bd28: 00000000fffffffe ffff88063d51bdd8
    ffff88063d51bd38: ffff880256518000 00000000ffffff9c
    ffff88063d51bd48: 0000000000000001 00007ff3c4889ef0
    ffff88063d51bd58: ffff88063d51bd98 ffffffff81172b4a
#10 [ffff88063d51bd60] path_walk at ffffffff81172b4a


Pourquoi CONFIG_DEBUG_LIST est positionne sur ce kernel ?

sh> x/a 0xffff88063d51bc98 - 0x8
0xffff88063d51bc90:     0xffff88063d51bd18
crash> struct path 0xffff88063d51bd18
struct path {
  mnt = 0xffff88033cf64cc0,
  dentry = 0xffff8804b0ede200
}
crash> kmem 0xffff8804b0ede200
CACHE            NAME                 OBJSIZE  ALLOCATED     TOTAL  SLABS  SSIZE
ffff88063fc33440 dentry                   192     896071   2032340  101617     4k
SLAB              MEMORY            TOTAL  ALLOCATED  FREE
ffff8804b0ede000  ffff8804b0ede080     20          8    12
FREE / [ALLOCATED]
  [ffff8804b0ede200]

      PAGE        PHYSICAL      MAPPING       INDEX CNT FLAGS
ffffea00106b4090 4b0ede000                0 7f5da65eb  1 c00000000000080
crash> struct nameidata 0xffff88063d51bdd8
struct nameidata {
  path = {
    mnt = 0xffff88033cf64cc0,
    dentry = 0xffff8804b0ede200
  },
  last = {
    hash = 2176636944,
    len = 4294967295,
    name = 0xffff88063ced8708 "\004"
  },
  root = {
    mnt = 0xffff88063e45ae80,
    dentry = 0xffff88063e1adec0
  },
  flags = 5,
  last_type = 1,
  depth = 0,
  saved_names = {0xe <Address 0xe out of bounds>, 0xffffea0005b3f738 "", 0xffff88063d51be78 "¸¾Q=\006\210ÿÿø¾Q=\006\210ÿÿ\200\234\210Äó\177", 0xffffffff81138cb4 "A)Üt\rHcÛM\215lÝ", 0xffff88063ced8708 "\004", 0xffff8801f91d43a0 "@\204í<\006\210ÿÿ", 0xffff88063ced8440 "xÌG<\006\210ÿÿPY\237<\003\210ÿÿ", 0xffff88002824cac0 "@\204í<\006\210ÿÿ", 0xffff88063d51bec8 "ø¾Q=\006\210ÿÿ`\236\210Äó\177"},
  intent = {
    open = {
      flags = 642,
      create_mode = 0,
      file = 0xffff88063ced8440
    }
  }
}
crash>

crash> struct vfsmount 0xffff88033cf64cc0
struct vfsmount {
  mnt_hash = {
    next = 0xffff88063e453f10,
    prev = 0xffff88063e453f10
  },
  mnt_parent = 0xffff88063e45ae80,
  mnt_mountpoint = 0xffff88063e196ec0,
  mnt_root = 0xffff88033e000bc0,
  mnt_sb = 0xffff88063e457000,
  mnt_mounts = {
    next = 0xffff88063cb71dc0,
    prev = 0xffff88063cb71dc0
  },
  mnt_child = {
    next = 0xffff88063e45adc0,
    prev = 0xffff88033cf64e00
  },
  mnt_flags = 32,
  rh_reserved = 0,
  rh_reserved2 = {
    first = 0x0
  },
  mnt_devname = 0xffff88033cfbc540 "/sys",
  mnt_list = {
    next = 0xffff88033cf64c28,
    prev = 0xffff88033cf64e28
  },
  mnt_expire = {
    next = 0xffff88033cf64d38,
    prev = 0xffff88033cf64d38
  },
  mnt_share = {
    next = 0xffff88033cf64d48,
    prev = 0xffff88033cf64d48
  },
  mnt_slave_list = {
    next = 0xffff88033cf64d58,
    prev = 0xffff88033cf64d58
  },
  mnt_slave = {
    next = 0xffff88033cf64d68,
    prev = 0xffff88033cf64d68
  },
  mnt_master = 0x0,
  mnt_ns = 0xffff88063fc04a00,
  mnt_id = 15,
  mnt_group_id = 0,
  mnt_count = {
    counter = 4
  },
  mnt_expiry_mark = 0,
  mnt_pinned = 0,
  mnt_ghosts = 0,
  mnt_writers = 0x60fcb50009d4
}

crash> p dcache_lock
dcache_lock = $3 = {
  raw_lock = {
    slock = 1179010628
  }
}


/* Drop dentry if it is not used already, unhash otherwise.
   Should be called with dcache lock held!
   Returns: 1 if dentry was dropped, 0 if unhashed. */
int ll_drop_dentry(struct dentry *dentry)
{
        lock_dentry(dentry);
        if (atomic_read(&dentry->d_count) == 0) {
                CDEBUG(D_DENTRY, "deleting dentry %.*s (%p) parent %p "
                       "inode %p\n", dentry->d_name.len,
                       dentry->d_name.name, dentry, dentry->d_parent,
                       dentry->d_inode);
                dget_locked(dentry);
                __d_drop(dentry);
                unlock_dentry(dentry);
                spin_unlock(&dcache_lock);
                cfs_spin_unlock(&ll_lookup_lock);
                dput(dentry);
                cfs_spin_lock(&ll_lookup_lock);
                spin_lock(&dcache_lock);
                return 1;
        }
        /* disconected dentry can not be find without lookup, because we
         * not need his to unhash or mark invalid. */
        if (dentry->d_flags & DCACHE_DISCONNECTED) {
                unlock_dentry(dentry);
                RETURN (0);
        }

        if (!(dentry->d_flags & DCACHE_LUSTRE_INVALID)) {
                CDEBUG(D_DENTRY, "unhashing dentry %.*s (%p) parent %p "
                       "inode %p refc %d\n", dentry->d_name.len,
                       dentry->d_name.name, dentry, dentry->d_parent,
                       dentry->d_inode, atomic_read(&dentry->d_count));
                /* actually we don't unhash the dentry, rather just
                 * mark it inaccessible for to __d_lookup(). otherwise
                 * sys_getcwd() could return -ENOENT -bzzz */
                dentry->d_flags |= DCACHE_LUSTRE_INVALID;
                if (!dentry->d_inode || !S_ISDIR(dentry->d_inode->i_mode))
                        __d_drop(dentry);
        }
        unlock_dentry(dentry);
        return 0;
}


crash> struct dentry.d_flags ffff8805d305a180
  d_flags = 24,  0x18  DCACHE_UNHASHED DCACHE_REFERENCED


[root@germain0 linux-2.6.32-220.23.1.bl6.Bull.28.8.x86_64]# grep CONFIG_DEBUG_LIST conf*
config-generic:CONFIG_DEBUG_LIST=y
[root@germain0 linux-2.6.32-220.23.1.bl6.Bull.28.8.x86_64]# grep CONFIG_DEBUG_LIST kernel-2.6.32-x86_64.config.ORIG
CONFIG_DEBUG_LIST=y

static inline void __d_drop(struct dentry *dentry)
{
    if (!(dentry->d_flags & DCACHE_UNHASHED)) {
        dentry->d_flags |= DCACHE_UNHASHED;
        hlist_del_rcu(&dentry->d_hash);
    }
}

static inline void hlist_del_rcu(struct hlist_node *n)
{
    __hlist_del(n);
    n->pprev = LIST_POISON2;
}

crash> disas ll_d_iput
Dump of assembler code for function ll_d_iput:
0xffffffffa09ed8a0 <ll_d_iput+0>:       push   %rbp
0xffffffffa09ed8a1 <ll_d_iput+1>:       mov    %rsp,%rbp
0xffffffffa09ed8a4 <ll_d_iput+4>:       push   %rbx
0xffffffffa09ed8a5 <ll_d_iput+5>:       sub    $0x8,%rsp
0xffffffffa09ed8a9 <ll_d_iput+9>:       nopl   0x0(%rax,%rax,1)
0xffffffffa09ed8ae <ll_d_iput+14>:      test   %rsi,%rsi
0xffffffffa09ed8b1 <ll_d_iput+17>:      mov    %rsi,%rbx
0xffffffffa09ed8b4 <ll_d_iput+20>:      je     0xffffffffa09ed8d8 <ll_d_iput+56>
0xffffffffa09ed8b6 <ll_d_iput+22>:      mov    %rsi,%rdi
0xffffffffa09ed8b9 <ll_d_iput+25>:      callq  0xffffffffa09eccb0 <find_cbdata>
0xffffffffa09ed8be <ll_d_iput+30>:      test   %eax,%eax
0xffffffffa09ed8c0 <ll_d_iput+32>:      jne    0xffffffffa09ed8c9 <ll_d_iput+41>
0xffffffffa09ed8c2 <ll_d_iput+34>:      movl   $0x0,0x4c(%rbx)
0xffffffffa09ed8c9 <ll_d_iput+41>:      mov    %rbx,%rdi
0xffffffffa09ed8cc <ll_d_iput+44>:      callq  0xffffffff8117d3e0 <iput>
0xffffffffa09ed8d1 <ll_d_iput+49>:      add    $0x8,%rsp
0xffffffffa09ed8d5 <ll_d_iput+53>:      pop    %rbx
0xffffffffa09ed8d6 <ll_d_iput+54>:      leaveq
0xffffffffa09ed8d7 <ll_d_iput+55>:      retq
0xffffffffa09ed8d8 <ll_d_iput+56>:      mov    $0xffffffffa0a64a60,%rdi
0xffffffffa09ed8df <ll_d_iput+63>:      mov    $0xffffffffa0a5e11c,%rdx
0xffffffffa09ed8e6 <ll_d_iput+70>:      mov    $0xffffffffa0a5e028,%rsi
0xffffffffa09ed8ed <ll_d_iput+77>:      xor    %eax,%eax
0xffffffffa09ed8ef <ll_d_iput+79>:      movl   $0x40000,0x7717f(%rip)        # 0xffffffffa0a64a78 <msgdata.79590+24>
0xffffffffa09ed8f9 <ll_d_iput+89>:      callq  0xffffffffa03d8050 <libcfs_debug_msg>
0xffffffffa09ed8fe <ll_d_iput+94>:      mov    $0xffffffffa0a64a20,%rdi
0xffffffffa09ed905 <ll_d_iput+101>:     movl   $0x40000,0x77129(%rip)        # 0xffffffffa0a64a38 <msgdata.79592+24>
0xffffffffa09ed90f <ll_d_iput+111>:     callq  0xffffffffa03cbdc0 <lbug_with_loc>
End of assembler dump.


crash> disas shrink_dcache_for_umount_subtree
Dump of assembler code for function shrink_dcache_for_umount_subtree:
0xffffffff8117b1f0 <shrink_dcache_for_umount_subtree+0>:        push   %rbp
0xffffffff8117b1f1 <shrink_dcache_for_umount_subtree+1>:        mov    %rsp,%rbp
0xffffffff8117b1f4 <shrink_dcache_for_umount_subtree+4>:        push   %r15
0xffffffff8117b1f6 <shrink_dcache_for_umount_subtree+6>:        push   %r14
0xffffffff8117b1f8 <shrink_dcache_for_umount_subtree+8>:        push   %r13
0xffffffff8117b1fa <shrink_dcache_for_umount_subtree+10>:       push   %r12
0xffffffff8117b1fc <shrink_dcache_for_umount_subtree+12>:       push   %rbx
0xffffffff8117b1fd <shrink_dcache_for_umount_subtree+13>:       sub    $0x18,%rsp
0xffffffff8117b201 <shrink_dcache_for_umount_subtree+17>:       nopl   0x0(%rax,%rax,1)
0xffffffff8117b206 <shrink_dcache_for_umount_subtree+22>:       cmp    0x28(%rdi),%rdi
0xffffffff8117b20a <shrink_dcache_for_umount_subtree+26>:       mov    %rdi,%rbx
0xffffffff8117b20d <shrink_dcache_for_umount_subtree+29>:       jne    0xffffffff8117b4bc <shrink_dcache_for_umount_subtree+716>
0xffffffff8117b213 <shrink_dcache_for_umount_subtree+35>:       mov    $0xffffffff81a052c0,%rdi
0xffffffff8117b21a <shrink_dcache_for_umount_subtree+42>:       callq  0xffffffff814875a0 <_spin_lock>
0xffffffff8117b21f <shrink_dcache_for_umount_subtree+47>:       mov    0x40(%rbx),%rdx
0xffffffff8117b223 <shrink_dcache_for_umount_subtree+51>:       lea    0x40(%rbx),%rax
0xffffffff8117b227 <shrink_dcache_for_umount_subtree+55>:       cmp    %rdx,%rax
0xffffffff8117b22a <shrink_dcache_for_umount_subtree+58>:       je     0xffffffff8117b254 <shrink_dcache_for_umount_subtree+100>
0xffffffff8117b22c <shrink_dcache_for_umount_subtree+60>:       mov    0x48(%rbx),%rcx
0xffffffff8117b230 <shrink_dcache_for_umount_subtree+64>:       mov    %rcx,0x8(%rdx)
0xffffffff8117b234 <shrink_dcache_for_umount_subtree+68>:       mov    %rdx,(%rcx)
0xffffffff8117b237 <shrink_dcache_for_umount_subtree+71>:       mov    %rax,0x40(%rbx)
0xffffffff8117b23b <shrink_dcache_for_umount_subtree+75>:       mov    %rax,0x48(%rbx)
0xffffffff8117b23f <shrink_dcache_for_umount_subtree+79>:       mov    0x90(%rbx),%rax
0xffffffff8117b246 <shrink_dcache_for_umount_subtree+86>:       subl   $0x1,0x108(%rax)
0xffffffff8117b24d <shrink_dcache_for_umount_subtree+93>:       subl   $0x1,0x8b8a10(%rip)        # 0xffffffff81a33c64
0xffffffff8117b254 <shrink_dcache_for_umount_subtree+100>:      mov    0x4(%rbx),%eax
0xffffffff8117b257 <shrink_dcache_for_umount_subtree+103>:      test   $0x10,%al
0xffffffff8117b259 <shrink_dcache_for_umount_subtree+105>:      jne    0xffffffff8117b283 <shrink_dcache_for_umount_subtree+147>
0xffffffff8117b25b <shrink_dcache_for_umount_subtree+107>:      or     $0x10,%eax
0xffffffff8117b25e <shrink_dcache_for_umount_subtree+110>:      mov    0x20(%rbx),%rdx
0xffffffff8117b262 <shrink_dcache_for_umount_subtree+114>:      mov    %eax,0x4(%rbx)
0xffffffff8117b265 <shrink_dcache_for_umount_subtree+117>:      mov    0x18(%rbx),%rax
0xffffffff8117b269 <shrink_dcache_for_umount_subtree+121>:      test   %rax,%rax
0xffffffff8117b26c <shrink_dcache_for_umount_subtree+124>:      mov    %rax,(%rdx)
0xffffffff8117b26f <shrink_dcache_for_umount_subtree+127>:      je     0xffffffff8117b275 <shrink_dcache_for_umount_subtree+133>
0xffffffff8117b271 <shrink_dcache_for_umount_subtree+129>:      mov    %rdx,0x8(%rax)
0xffffffff8117b275 <shrink_dcache_for_umount_subtree+133>:      mov    $0xdead000000200200,%rcx
0xffffffff8117b27f <shrink_dcache_for_umount_subtree+143>:      mov    %rcx,0x20(%rbx)
0xffffffff8117b283 <shrink_dcache_for_umount_subtree+147>:      mov    $0xffffffff81a052c0,%r13
0xffffffff8117b28a <shrink_dcache_for_umount_subtree+154>:      mov    %r13,%rax
0xffffffff8117b28d <shrink_dcache_for_umount_subtree+157>:      incw   (%rax)
0xffffffff8117b290 <shrink_dcache_for_umount_subtree+160>:      xchg   %ax,%ax
0xffffffff8117b293 <shrink_dcache_for_umount_subtree+163>:      xor    %r12d,%r12d
0xffffffff8117b296 <shrink_dcache_for_umount_subtree+166>:      nopw   %cs:0x0(%rax,%rax,1)
0xffffffff8117b2a0 <shrink_dcache_for_umount_subtree+176>:      lea    0x60(%rbx),%r15
0xffffffff8117b2a4 <shrink_dcache_for_umount_subtree+180>:      cmp    0x60(%rbx),%r15
0xffffffff8117b2a8 <shrink_dcache_for_umount_subtree+184>:      je     0xffffffff8117b392 <shrink_dcache_for_umount_subtree+418>
0xffffffff8117b2ae <shrink_dcache_for_umount_subtree+190>:      mov    $0xffffffff81a052c0,%rdi
0xffffffff8117b2b5 <shrink_dcache_for_umount_subtree+197>:      callq  0xffffffff814875a0 <_spin_lock>
0xffffffff8117b2ba <shrink_dcache_for_umount_subtree+202>:      mov    0x60(%rbx),%rax
0xffffffff8117b2be <shrink_dcache_for_umount_subtree+206>:      cmp    %rax,%r15
0xffffffff8117b2c1 <shrink_dcache_for_umount_subtree+209>:      lea    -0x50(%rax),%r14
0xffffffff8117b2c5 <shrink_dcache_for_umount_subtree+213>:      je     0xffffffff8117b34f <shrink_dcache_for_umount_subtree+351>
0xffffffff8117b2cb <shrink_dcache_for_umount_subtree+219>:      nopl   0x0(%rax,%rax,1)
0xffffffff8117b2d0 <shrink_dcache_for_umount_subtree+224>:      mov    0x40(%r14),%rdx
0xffffffff8117b2d4 <shrink_dcache_for_umount_subtree+228>:      lea    0x40(%r14),%rax
0xffffffff8117b2d8 <shrink_dcache_for_umount_subtree+232>:      cmp    %rdx,%rax
0xffffffff8117b2db <shrink_dcache_for_umount_subtree+235>:      je     0xffffffff8117b305 <shrink_dcache_for_umount_subtree+277>
0xffffffff8117b2dd <shrink_dcache_for_umount_subtree+237>:      mov    0x48(%r14),%rcx
0xffffffff8117b2e1 <shrink_dcache_for_umount_subtree+241>:      mov    %rcx,0x8(%rdx)
0xffffffff8117b2e5 <shrink_dcache_for_umount_subtree+245>:      mov    %rdx,(%rcx)
0xffffffff8117b2e8 <shrink_dcache_for_umount_subtree+248>:      mov    %rax,0x40(%r14)
0xffffffff8117b2ec <shrink_dcache_for_umount_subtree+252>:      mov    %rax,0x48(%r14)
0xffffffff8117b2f0 <shrink_dcache_for_umount_subtree+256>:      mov    0x90(%r14),%rax
0xffffffff8117b2f7 <shrink_dcache_for_umount_subtree+263>:      subl   $0x1,0x108(%rax)
0xffffffff8117b2fe <shrink_dcache_for_umount_subtree+270>:      subl   $0x1,0x8b895f(%rip)        # 0xffffffff81a33c64
0xffffffff8117b305 <shrink_dcache_for_umount_subtree+277>:      mov    0x4(%r14),%eax
0xffffffff8117b309 <shrink_dcache_for_umount_subtree+281>:      test   $0x10,%al
0xffffffff8117b30b <shrink_dcache_for_umount_subtree+283>:      jne    0xffffffff8117b336 <shrink_dcache_for_umount_subtree+326>
0xffffffff8117b30d <shrink_dcache_for_umount_subtree+285>:      or     $0x10,%eax
0xffffffff8117b310 <shrink_dcache_for_umount_subtree+288>:      mov    0x20(%r14),%rdx
0xffffffff8117b314 <shrink_dcache_for_umount_subtree+292>:      mov    %eax,0x4(%r14)
0xffffffff8117b318 <shrink_dcache_for_umount_subtree+296>:      mov    0x18(%r14),%rax
0xffffffff8117b31c <shrink_dcache_for_umount_subtree+300>:      test   %rax,%rax
0xffffffff8117b31f <shrink_dcache_for_umount_subtree+303>:      mov    %rax,(%rdx)
0xffffffff8117b322 <shrink_dcache_for_umount_subtree+306>:      je     0xffffffff8117b328 <shrink_dcache_for_umount_subtree+312>
0xffffffff8117b324 <shrink_dcache_for_umount_subtree+308>:      mov    %rdx,0x8(%rax)
0xffffffff8117b328 <shrink_dcache_for_umount_subtree+312>:      mov    $0xdead000000200200,%rdx
0xffffffff8117b332 <shrink_dcache_for_umount_subtree+322>:      mov    %rdx,0x20(%r14)
0xffffffff8117b336 <shrink_dcache_for_umount_subtree+326>:      mov    $0xffffffff81a052c0,%rdi
0xffffffff8117b33d <shrink_dcache_for_umount_subtree+333>:      callq  0xffffffff8104ead0 <__cond_resched_lock>
0xffffffff8117b342 <shrink_dcache_for_umount_subtree+338>:      mov    0x50(%r14),%rax
0xffffffff8117b346 <shrink_dcache_for_umount_subtree+342>:      cmp    %rax,%r15
0xffffffff8117b349 <shrink_dcache_for_umount_subtree+345>:      lea    -0x50(%rax),%r14
0xffffffff8117b34d <shrink_dcache_for_umount_subtree+349>:      jne    0xffffffff8117b2d0 <shrink_dcache_for_umount_subtree+224>
0xffffffff8117b34f <shrink_dcache_for_umount_subtree+351>:      mov    %r13,%rax
0xffffffff8117b352 <shrink_dcache_for_umount_subtree+354>:      incw   (%rax)
0xffffffff8117b355 <shrink_dcache_for_umount_subtree+357>:      xchg   %ax,%ax
0xffffffff8117b358 <shrink_dcache_for_umount_subtree+360>:      mov    0x60(%rbx),%rbx
0xffffffff8117b35c <shrink_dcache_for_umount_subtree+364>:      sub    $0x50,%rbx
0xffffffff8117b360 <shrink_dcache_for_umount_subtree+368>:      jmpq   0xffffffff8117b2a0 <shrink_dcache_for_umount_subtree+176>
0xffffffff8117b365 <shrink_dcache_for_umount_subtree+373>:      nopl   (%rax)
0xffffffff8117b368 <shrink_dcache_for_umount_subtree+376>:      mov    %rbx,%rdi    rbx = struct dentry * de maybe ??? 
0xffffffff8117b36b <shrink_dcache_for_umount_subtree+379>:      callq  *%rax        Call  ll_d_iput
0xffffffff8117b36d <shrink_dcache_for_umount_subtree+381>:      mov    %rbx,%rdi    
0xffffffff8117b370 <shrink_dcache_for_umount_subtree+384>:      callq  0xffffffff81179e70 <d_free>
0xffffffff8117b375 <shrink_dcache_for_umount_subtree+389>:      test   %r14,%r14
0xffffffff8117b378 <shrink_dcache_for_umount_subtree+392>:      je     0xffffffff8117b43a <shrink_dcache_for_umount_subtree+586>
0xffffffff8117b37e <shrink_dcache_for_umount_subtree+398>:      mov    0x60(%r14),%rax
0xffffffff8117b382 <shrink_dcache_for_umount_subtree+402>:      lea    0x60(%r14),%rdx
0xffffffff8117b386 <shrink_dcache_for_umount_subtree+406>:      mov    %r14,%rbx
0xffffffff8117b389 <shrink_dcache_for_umount_subtree+409>:      cmp    %rdx,%rax
0xffffffff8117b38c <shrink_dcache_for_umount_subtree+412>:      jne    0xffffffff8117b470 <shrink_dcache_for_umount_subtree+640>
0xffffffff8117b392 <shrink_dcache_for_umount_subtree+418>:      mov    (%rbx),%eax
0xffffffff8117b394 <shrink_dcache_for_umount_subtree+420>:      test   %eax,%eax
0xffffffff8117b396 <shrink_dcache_for_umount_subtree+422>:      jne    0xffffffff8117b479 <shrink_dcache_for_umount_subtree+649>
0xffffffff8117b39c <shrink_dcache_for_umount_subtree+428>:      mov    0x28(%rbx),%r14
0xffffffff8117b3a0 <shrink_dcache_for_umount_subtree+432>:      cmp    %rbx,%r14
0xffffffff8117b3a3 <shrink_dcache_for_umount_subtree+435>:      je     0xffffffff8117b468 <shrink_dcache_for_umount_subtree+632>
0xffffffff8117b3a9 <shrink_dcache_for_umount_subtree+441>:      lock decl (%r14)
0xffffffff8117b3ad <shrink_dcache_for_umount_subtree+445>:      mov    0x58(%rbx),%rax
0xffffffff8117b3b1 <shrink_dcache_for_umount_subtree+449>:      mov    0x50(%rbx),%rdx
0xffffffff8117b3b5 <shrink_dcache_for_umount_subtree+453>:      mov    $0xdead000000200200,%r15
0xffffffff8117b3bf <shrink_dcache_for_umount_subtree+463>:      add    $0x1,%r12d
0xffffffff8117b3c3 <shrink_dcache_for_umount_subtree+467>:      mov    %rax,0x8(%rdx)
0xffffffff8117b3c7 <shrink_dcache_for_umount_subtree+471>:      mov    %rdx,(%rax)
0xffffffff8117b3ca <shrink_dcache_for_umount_subtree+474>:      mov    $0xdead000000100100,%rax
0xffffffff8117b3d4 <shrink_dcache_for_umount_subtree+484>:      mov    0x10(%rbx),%rsi
0xffffffff8117b3d8 <shrink_dcache_for_umount_subtree+488>:      mov    %rax,0x50(%rbx)
0xffffffff8117b3dc <shrink_dcache_for_umount_subtree+492>:      mov    %r15,0x58(%rbx)
0xffffffff8117b3e0 <shrink_dcache_for_umount_subtree+496>:      test   %rsi,%rsi
0xffffffff8117b3e3 <shrink_dcache_for_umount_subtree+499>:      je     0xffffffff8117b36d <shrink_dcache_for_umount_subtree+381>
0xffffffff8117b3e5 <shrink_dcache_for_umount_subtree+501>:      mov    0x70(%rbx),%rcx
0xffffffff8117b3e9 <shrink_dcache_for_umount_subtree+505>:      mov    0x78(%rbx),%rdx
0xffffffff8117b3ed <shrink_dcache_for_umount_subtree+509>:      lea    0x70(%rbx),%rax
0xffffffff8117b3f1 <shrink_dcache_for_umount_subtree+513>:      movq   $0x0,0x10(%rbx)
0xffffffff8117b3f9 <shrink_dcache_for_umount_subtree+521>:      mov    %rdx,0x8(%rcx)
0xffffffff8117b3fd <shrink_dcache_for_umount_subtree+525>:      mov    %rcx,(%rdx)
0xffffffff8117b400 <shrink_dcache_for_umount_subtree+528>:      mov    %rax,0x70(%rbx)
0xffffffff8117b404 <shrink_dcache_for_umount_subtree+532>:      mov    %rax,0x78(%rbx)
0xffffffff8117b408 <shrink_dcache_for_umount_subtree+536>:      mov    0x88(%rbx),%rax
0xffffffff8117b40f <shrink_dcache_for_umount_subtree+543>:      test   %rax,%rax
0xffffffff8117b412 <shrink_dcache_for_umount_subtree+546>:      je     0xffffffff8117b421 <shrink_dcache_for_umount_subtree+561>
0xffffffff8117b414 <shrink_dcache_for_umount_subtree+548>:      mov    0x28(%rax),%rax
0xffffffff8117b418 <shrink_dcache_for_umount_subtree+552>:      test   %rax,%rax
0xffffffff8117b41b <shrink_dcache_for_umount_subtree+555>:      jne    0xffffffff8117b368 <shrink_dcache_for_umount_subtree+376>
0xffffffff8117b421 <shrink_dcache_for_umount_subtree+561>:      mov    %rsi,%rdi
0xffffffff8117b424 <shrink_dcache_for_umount_subtree+564>:      callq  0xffffffff8117d3e0 <iput>
0xffffffff8117b429 <shrink_dcache_for_umount_subtree+569>:      mov    %rbx,%rdi
0xffffffff8117b42c <shrink_dcache_for_umount_subtree+572>:      callq  0xffffffff81179e70 <d_free>
0xffffffff8117b431 <shrink_dcache_for_umount_subtree+577>:      test   %r14,%r14
0xffffffff8117b434 <shrink_dcache_for_umount_subtree+580>:      jne    0xffffffff8117b37e <shrink_dcache_for_umount_subtree+398>
0xffffffff8117b43a <shrink_dcache_for_umount_subtree+586>:      mov    $0xffffffff81a052c0,%rdi
0xffffffff8117b441 <shrink_dcache_for_umount_subtree+593>:      callq  0xffffffff814875a0 <_spin_lock>
0xffffffff8117b446 <shrink_dcache_for_umount_subtree+598>:      sub    %r12d,0x8b8813(%rip)        # 0xffffffff81a33c60
0xffffffff8117b44d <shrink_dcache_for_umount_subtree+605>:      mov    %r13,%rax
0xffffffff8117b450 <shrink_dcache_for_umount_subtree+608>:      incw   (%rax)
0xffffffff8117b453 <shrink_dcache_for_umount_subtree+611>:      xchg   %ax,%ax
0xffffffff8117b456 <shrink_dcache_for_umount_subtree+614>:      add    $0x18,%rsp
0xffffffff8117b45a <shrink_dcache_for_umount_subtree+618>:      pop    %rbx
0xffffffff8117b45b <shrink_dcache_for_umount_subtree+619>:      pop    %r12
0xffffffff8117b45d <shrink_dcache_for_umount_subtree+621>:      pop    %r13
0xffffffff8117b45f <shrink_dcache_for_umount_subtree+623>:      pop    %r14
0xffffffff8117b461 <shrink_dcache_for_umount_subtree+625>:      pop    %r15
0xffffffff8117b463 <shrink_dcache_for_umount_subtree+627>:      leaveq
0xffffffff8117b464 <shrink_dcache_for_umount_subtree+628>:      retq
0xffffffff8117b465 <shrink_dcache_for_umount_subtree+629>:      nopl   (%rax)
0xffffffff8117b468 <shrink_dcache_for_umount_subtree+632>:      xor    %r14d,%r14d
0xffffffff8117b46b <shrink_dcache_for_umount_subtree+635>:      jmpq   0xffffffff8117b3ad <shrink_dcache_for_umount_subtree+445>
0xffffffff8117b470 <shrink_dcache_for_umount_subtree+640>:      lea    -0x50(%rax),%rbx
0xffffffff8117b474 <shrink_dcache_for_umount_subtree+644>:      jmpq   0xffffffff8117b2a0 <shrink_dcache_for_umount_subtree+176>
0xffffffff8117b479 <shrink_dcache_for_umount_subtree+649>:      mov    0x90(%rbx),%rax
0xffffffff8117b480 <shrink_dcache_for_umount_subtree+656>:      mov    0x10(%rbx),%rsi
0xffffffff8117b484 <shrink_dcache_for_umount_subtree+660>:      mov    (%rbx),%r8d
0xffffffff8117b487 <shrink_dcache_for_umount_subtree+663>:      mov    0x38(%rbx),%rcx
0xffffffff8117b48b <shrink_dcache_for_umount_subtree+667>:      mov    0x30(%rax),%rdx
0xffffffff8117b48f <shrink_dcache_for_umount_subtree+671>:      mov    (%rdx),%r9
0xffffffff8117b492 <shrink_dcache_for_umount_subtree+674>:      xor    %edx,%edx
0xffffffff8117b494 <shrink_dcache_for_umount_subtree+676>:      test   %rsi,%rsi
0xffffffff8117b497 <shrink_dcache_for_umount_subtree+679>:      je     0xffffffff8117b49d <shrink_dcache_for_umount_subtree+685>
0xffffffff8117b499 <shrink_dcache_for_umount_subtree+681>:      mov    0x40(%rsi),%rdx
0xffffffff8117b49d <shrink_dcache_for_umount_subtree+685>:      add    $0x270,%rax
0xffffffff8117b4a3 <shrink_dcache_for_umount_subtree+691>:      mov    %rbx,%rsi
0xffffffff8117b4a6 <shrink_dcache_for_umount_subtree+694>:      mov    $0xffffffff81777d10,%rdi
0xffffffff8117b4ad <shrink_dcache_for_umount_subtree+701>:      mov    %rax,(%rsp)
0xffffffff8117b4b1 <shrink_dcache_for_umount_subtree+705>:      xor    %eax,%eax
0xffffffff8117b4b3 <shrink_dcache_for_umount_subtree+707>:      callq  0xffffffff8148471b <printk>
0xffffffff8117b4b8 <shrink_dcache_for_umount_subtree+712>:      ud2a
0xffffffff8117b4ba <shrink_dcache_for_umount_subtree+714>:      jmp    0xffffffff8117b4ba <shrink_dcache_for_umount_subtree+714>
0xffffffff8117b4bc <shrink_dcache_for_umount_subtree+716>:      ud2a
0xffffffff8117b4be <shrink_dcache_for_umount_subtree+718>:      jmp    0xffffffff8117b4be <shrink_dcache_for_umount_subtree+718>
End of assembler dump.

#14 [ffff880120b2bdb0] ll_d_iput at ffffffffa09ed8be [lustre]
    ffff880120b2bdb8: 0000000000000246 ffff880593ec4600                <-- ici ffff880593ec4600 = rbx
    ffff880120b2bdc8: ffff880120b2be18 ffffffff8117b36d
#15 [ffff880120b2bdd0] shrink_dcache_for_umount_subtree at ffffffff8117b36d
    ffff880120b2bdd8: ffff8802d9506000 ffff8802d9506000
    ffff880120b2bde8: ffff880120b2bf38 ffff88033cd03000
    ffff880120b2bdf8: ffffffffa0a7e520 ffffffff81ac4940
    ffff880120b2be08: ffff88033cd03000 ffff88033d2c1850
    ffff880120b2be18: ffff880120b2be38 ffffffff8117b4f6
#16 [ffff880120b2be20] shrink_dcache_for_umount at ffffffff8117b4f6

crash> struct dentry ffff880593ec4600
struct dentry {
  d_count = {
    counter = 0
  },
  d_flags = 24,
  d_lock = {
    raw_lock = {
      slock = 327685
    }
  },
  d_mounted = 1768293427,
  d_inode = 0x0,
  d_hash = {
    next = 0x0,
    pprev = 0xdead000000200200
  },
  d_parent = 0xffff88053dd10180,
  d_name = {
    hash = 302013360,
    len = 25,
    name = 0xffff880593ec46a0 "[0x2000abc55:0x13886:0x0]"
  },
  d_lru = {
    next = 0xffff880593ec4640,
    prev = 0xffff880593ec4640
  },
  d_u = {
    d_child = {
      next = 0xdead000000100100,
      prev = 0xdead000000200200
    },
    d_rcu = {
      next = 0xdead000000100100,
      func = 0xdead000000200200
    }
  },
  d_subdirs = {
    next = 0xffff880593ec4660,
    prev = 0xffff880593ec4660
  },
  d_alias = {
    next = 0xffff880593ec4670,
    prev = 0xffff880593ec4670
  },
  d_time = 18446612139194222114,
  d_op = 0xffffffffa0a64140,
  d_sb = 0xffff88033cd03000,
  d_fsdata = 0xffff8804ed89bf40,
  d_iname = "[0x2000abc55:0x13886:0x0]\000\000\t®\033&{"
}

crash> struct dentry.d_parent ffff880593ec4600
  d_parent = 0xffff88053dd10180,
crash> struct dentry.d_parent ffff8805d305a180
  d_parent = 0xffff88053dd10180,
crash> struct dentry.d_subdirs 0xffff88053dd10180
  d_subdirs = {
    next = 0xffff880356ad4c10,
    prev = 0xffff880600534190
  },

crash> struct dentry ffff8803d633e6c0
struct dentry {
  d_count = {
    counter = 0
  },
  d_flags = 24,
  d_lock = {
    raw_lock = {
      slock = 327685
    }
  },
  d_mounted = 0,
  d_inode = 0x0,
  d_hash = {
    next = 0x0,
    pprev = 0xdead000000200200
  },
  d_parent = 0xffff88053dd10180,
  d_name = {
    hash = 92564698,
    len = 23,
    name = 0xffff8803d633e760 "[0x2000aba7f:0xf31:0x0]"
  },
  d_lru = {
    next = 0xffff8803d633e700,
    prev = 0xffff8803d633e700
  },
  d_u = {
    d_child = {
      next = 0xffff880036c96e90,
      prev = 0xffffffff8117a770
    },
    d_rcu = {
      next = 0xffff880036c96e90,
      func = 0xffffffff8117a770 <d_callback>
    }
  },
  d_subdirs = {
    next = 0xffff8803d633e720,
    prev = 0xffff8803d633e720
  },
  d_alias = {
    next = 0xffff8803d633e730,
    prev = 0xffff8803d633e730
  },
  d_time = 0,
  d_op = 0xffffffffa0a64140,
  d_sb = 0xffff88033cd03000,
  d_fsdata = 0xdeadbeef,
  d_iname = "[0x2000aba7f:0xf31:0x0]\000]\000\000\000\000\000\000"
}


static void __d_rehash(struct dentry * entry, struct hlist_head *list)
{

    entry->d_flags &= ~DCACHE_UNHASHED;
    hlist_add_head_rcu(&entry->d_hash, list);
}


dG

here is my scenario point of view :

 CPU 7 run thread ldlm_bl_44 (pid 26203) on ll_drop_dentry(0xffff8805d305a180)  and 
       have the dcache_lock and do the __d_drop(0xffff8805d305a180) ;
 CPU 0 run thread umount (pid 10428) and wait dcache_lock to do loop one_dentry.d_u.d_child on shrink_dcache_for_umount_subtree ()
 CPU 7 run thread ldlm_bl_44 (pid 26203) on ll_drop_dentry(0xffff8805d305a180) unlock dcache_lock and run dput(dentry); and wait dcache_lock
 CPU 0 run thread umount (pid 10428) take dcache_lock 
                                     do loop one_dentry.d_u.d_child 
                                     unlock dcache_lock
                                     mov 0xffff8805d305a180 as first child
                                     do list_del(&dentry->d_u.d_child); (put LIST_POISON1 on 0xffff8805d305a180->d_u.d_child 
 CPU 7 run thread ldlm_bl_44 (pid 26203) on dput(0xffff8805d305a180); have the dcache_lock do d_kill(0xffff8805d305a180)
                                     and try to do list_del(&0xffff8805d305a180->d_u.d_child); 
                                     and do generale_protection due to LIST_POISON1
 CPU 0 run thread umount (pid 10428) d_free(dentry) and put the dentry on the rcu queue via 

crash> log | grep RCU
Hierarchical RCU implementation.

crash> p rcu_sched_data
PER-CPU DATA TYPE:
  struct rcu_data per_cpu__rcu_sched_data;
PER-CPU ADDRESSES:
  [0]: ffff88002820e520
  [1]: ffff88002822e520
  [2]: ffff88002824e520
  [3]: ffff88002826e520
  [4]: ffff88034ac0e520
  [5]: ffff88034ac2e520
  [6]: ffff88034ac4e520
  [7]: ffff88034ac6e520
crash> p rcu_bh_data
PER-CPU DATA TYPE:
  struct rcu_data per_cpu__rcu_bh_data;
PER-CPU ADDRESSES:
  [0]: ffff88002820e620
  [1]: ffff88002822e620
  [2]: ffff88002824e620
  [3]: ffff88002826e620
  [4]: ffff88034ac0e620
  [5]: ffff88034ac2e620
  [6]: ffff88034ac4e620
  [7]: ffff88034ac6e620

crash> struct rcu_data ffff88034ac6e520
struct rcu_data {
  completed = 14982999,
  gpnum = 14983000,
  passed_quiesc_completed = 14982998,
  passed_quiesc = false,
  qs_pending = true,
  beenonline = true,
  preemptable = false,
  mynode = 0xffffffff81a25b80,
  grpmask = 128,
  nxtlist = 0xffff88041581d7d0,
  nxttail = {0xffff88034ac6e550, 0xffff88041581d950, 0xffff88041581d950, 0xffff88004c54ddd0},
  qlen = 53,
  qlen_last_fqs_check = 0,
  n_force_qs_snap = 16458536,
  blimit = 10,
  dynticks = 0xffff88034ac6e500,
  dynticks_snap = 104677663,
  dynticks_nmi_snap = 90,
  dynticks_fqs = 10726002,
  offline_fqs = 0,
  resched_ipi = 157677,
  n_rcu_pending = 18214994,
  n_rp_qs_pending = 6971682,
  n_rp_cb_ready = 20,
  n_rp_cpu_needs_gp = 80680,
  n_rp_gp_completed = 3070686,
  n_rp_gp_started = 20,
  n_rp_need_fqs = 388966,
  n_rp_need_nothing = 7702940,
  cpu = 7
}
crash> struct rcu_data ffff88034ac6e620
struct rcu_data {
  completed = 54077,
  gpnum = 54078,
  passed_quiesc_completed = 54077,
  passed_quiesc = true,
  qs_pending = false,
  beenonline = true,
  preemptable = false,
  mynode = 0xffffffff81a26940,
  grpmask = 128,
  nxtlist = 0x0,
  nxttail = {0xffff88034ac6e650, 0xffff88034ac6e650, 0xffff88034ac6e650, 0xffff88034ac6e650},
  qlen = 0,
  qlen_last_fqs_check = 0,
  n_force_qs_snap = 0,
  blimit = 10,
  dynticks = 0xffff88034ac6e500,
  dynticks_snap = 104677504,
  dynticks_nmi_snap = 90,
  dynticks_fqs = 41090,
  offline_fqs = 0,
  resched_ipi = 0,
  n_rcu_pending = 7702940,
  n_rp_qs_pending = 4463865,
  n_rp_cb_ready = 0,
  n_rp_cpu_needs_gp = 37,
  n_rp_gp_completed = 3231168,
  n_rp_gp_started = 0,
  n_rp_need_fqs = 367,
  n_rp_need_nothing = 7503,
  cpu = 7
}

crash> struct rcu_data ffff88002820e520
struct rcu_data {
  completed = 14982999,
  gpnum = 14983000,
  passed_quiesc_completed = 14982999,
  passed_quiesc = true,
  qs_pending = false,
  beenonline = true,
  preemptable = false,
  mynode = 0xffffffff81a25b80,
  grpmask = 1,
  nxtlist = 0xffff8806035ec110,
  nxttail = {0xffff88002820e550, 0xffff880602735d10, 0xffff8801f39dbe90, 0xffff880356ad4e50},
  qlen = 212058,
  qlen_last_fqs_check = 210021,
  n_force_qs_snap = 16986119,
  blimit = 9223372036854775807,
  dynticks = 0xffff88002820e500,
  dynticks_snap = 498499353,
  dynticks_nmi_snap = 626,
  dynticks_fqs = 7330766,
  offline_fqs = 0,
  resched_ipi = 33896,
  n_rcu_pending = 32999528,
  n_rp_qs_pending = 12664261,
  n_rp_cb_ready = 20,
  n_rp_cpu_needs_gp = 95363,
  n_rp_gp_completed = 4343842,
  n_rp_gp_started = 98,
  n_rp_need_fqs = 3278103,
  n_rp_need_nothing = 12617841,
  cpu = 0
}

crash> list 0xffff8806035ec110 | wc -l
212058
crash> list 0xffff8806035ec110 > list_nxtlist.txt
crash> !grep -B 1 ffff8803d633e710 list_nxtlist.txt
ffff8805d305a1d0
ffff8803d633e710
crash> x/a 0xffff8805d305a1d0 - 0x50
0xffff8805d305a180:     0x1800000000
crash> 
  vi list_nxtlist.txt

      1 ffff8806035ec110
      2 ffff8801d999a610
      3 ffff8806035ece90
      4 ffff8801d999a3d0
      5 ffff8801d999a310
  .......
  92330 ffff88041479e3d0
  92331 ffff880036c96ad0
  92332 ffff8805d305a1d0    <--- dentry managed by thread ldlm_bl_44
  92333 ffff8803d633e710
  92334 ffff880036c96e90
  92335 ffff88041479d3d0
  .......
 212053 ffff88029c825c10
 212054 ffff8804de909c10
 212055 ffff8804de909490
 212056 ffff880593ec41d0
 212057 ffff880274215e90
 212058 ffff880356ad4e50



Resume:

Race condition between shrink_dcache_for_umount_subtree()(umount) and dput()/d_kill() (ldlm_bl)

general_protection on d_kill on because LIST_POISON1 on RDX

crash> bt 26203
PID: 26203  TASK: ffff88033d5dc040  CPU: 7   COMMAND: "ldlm_bl_44"
 #0 [ffff8802e12e7ab0] machine_kexec at ffffffff8102895b
 #1 [ffff8802e12e7b10] crash_kexec at ffffffff810a4622
 #2 [ffff8802e12e7be0] oops_end at ffffffff814886c0
 #3 [ffff8802e12e7c10] die at ffffffff8100714b
 #4 [ffff8802e12e7c40] do_general_protection at ffffffff81488252
 #5 [ffff8802e12e7c70] general_protection at ffffffff81487a25
    [exception RIP: d_kill+53]
    RIP: ffffffff81179f05  RSP: ffff8802e12e7d20  RFLAGS: 00010282
    RAX: dead000000200200  RBX: ffff8805d305a180  RCX: ffff8805d305a1c0
    RDX: dead000000100100  RSI: dead000000200200  RDI: dead000000100100
    RBP: ffff8802e12e7d30   R8: 00000000deadbeef   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 5a5a5a5a5a5a5a5a  R12: ffff8805d305a188
    R13: ffffffffa0a848a0  R14: ffff88053dd101e0  R15: ffff8804d9652af8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #6 [ffff8802e12e7d38] dput at ffffffff8117b971
 #7 [ffff8802e12e7d58] ll_drop_dentry at ffffffffa09eb626 [lustre]
 #8 [ffff8802e12e7d98] ll_md_blocking_ast at ffffffffa0a2c702 [lustre]
 #9 [ffff8802e12e7e08] ldlm_cancel_callback at ffffffffa05f01f0 [ptlrpc]
#10 [ffff8802e12e7e28] ldlm_cli_cancel_local at ffffffffa060a20b [ptlrpc]
#11 [ffff8802e12e7e58] ldlm_cli_cancel_list_local at ffffffffa060cafd [ptlrpc]
#12 [ffff8802e12e7eb8] ldlm_bl_thread_main at ffffffffa06114f0 [ptlrpc]
#13 [ffff8802e12e7f48] kernel_thread at ffffffff8100412a

0xffffffff81179edc <d_kill+12>: mov    %r12,0x8(%rsp)
0xffffffff81179ee1 <d_kill+17>: nopl   0x0(%rax,%rax,1)
0xffffffff81179ee6 <d_kill+22>: mov    0x50(%rdi),%rdx     <--- rdi = 0xffff8805d305a180=rbx  struct dentry.d_u.d_child.next
0xffffffff81179eea <d_kill+26>: mov    0x58(%rdi),%rax     <----rdi = 0xffff8805d305a180=rbx  struct dentry.d_u.d_child.prev
0xffffffff81179eee <d_kill+30>: mov    $0xdead000000200200,%rsi   LIST_POISON2
0xffffffff81179ef8 <d_kill+40>: mov    %rdi,%rbx                  rbx = *(struct dentry *) 
0xffffffff81179efb <d_kill+43>: mov    $0xdead000000100100,%rdi   LIST_POISON1
0xffffffff81179f05 <d_kill+53>: mov    %rax,0x8(%rdx)      <--- general_protection  RDX: dead000000100100 is LIST_POISON1 rax=dead000000200200 (LIST_POISON2)
0xffffffff81179f09 <d_kill+57>: mov    %rdx,(%rax)
0xffffffff81179f0c <d_kill+60>: mov    %rdi,0x50(%rbx)

crash> struct dentry ffff8805d305a180
struct dentry {
  d_count = {
    counter = 0
  },
  d_flags = 24,
  d_lock = {
    raw_lock = {
      slock = 458758
    }
  },
  d_mounted = 0,
  d_inode = 0x0,
  d_hash = {
    next = 0x0,
    pprev = 0xdead000000200200
  },
  d_parent = 0xffff88053dd10180,
  d_name = {
    hash = 3302437006,
    len = 24,
    name = 0xffff8805d305a220 "[0x2000abc6b:0x10bb:0x0]"
  },
  d_lru = {
    next = 0xffff8805d305a1c0,
    prev = 0xffff8805d305a1c0
  },
  d_u = {
    d_child = {
      next = 0xffff8803d633e710,    
      prev = 0xffffffff8117a770    
    },
    d_rcu = {
      next = 0xffff8803d633e710,  <--- the dentry ffff8805d305a180 was put on the rcu queue
      func = 0xffffffff8117a770 <d_callback>
    }
  },
  d_subdirs = {
    next = 0xffff8805d305a1e0,
    prev = 0xffff8805d305a1e0
  },
  d_alias = {
    next = 0xffff8805d305a1f0,
    prev = 0xffff8805d305a1f0
  },
  d_time = 0,
  d_op = 0xffffffffa0a64140,
  d_sb = 0xffff88033cd03000,
  d_fsdata = 0xdeadbeef,
  d_iname = "[0x2000abc6b:0x10bb:0x0]\000\000\000\000\000\000\000"
}

On THIS dentry.d-u pointer, now, the "poison" pointer has desapear 
and now this dentry is on the RCU list to be freed on the next 
quiesed period.
So there are an another thread who have manipulate this dentry during the same time :

crash> bt 10428
PID: 10428  TASK: ffff88033d2c1850  CPU: 0   COMMAND: "umount"
 #0 [ffff880028207e90] crash_nmi_callback at ffffffff81020b56
 #1 [ffff880028207ea0] notifier_call_chain at ffffffff8148a755
 #2 [ffff880028207ee0] atomic_notifier_call_chain at ffffffff8148a7ba
 #3 [ffff880028207ef0] notify_die at ffffffff810817ce
 #4 [ffff880028207f20] do_nmi at ffffffff814883d3
 #5 [ffff880028207f50] nmi at ffffffff81487ce0
    [exception RIP: ldlm_res_hop_fid_hash+25]
    RIP: ffffffffa05f43b9  RSP: ffff880120b2bb98  RFLAGS: 00000292
    RAX: ffffffffa06acda0  RBX: ffff8805f7cbc300  RCX: 0000000000000010
    RDX: 000000000000ffff  RSI: ffff880120b2bcc8  RDI: ffff8805f7cbc300
    RBP: ffff880120b2bbd8   R8: ffff880120b2bc48   R9: 5a5a5a5a5a5a5a5a
    R10: 5a5a5a5a5a5a5a5a  R11: 00000002000abc55  R12: 0000000000000010
    R13: ffff8805f8216bc0  R14: ffff880120b2bc48  R15: 0000000000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
--- <NMI exception stack> ---
 #6 [ffff880120b2bb98] ldlm_res_hop_fid_hash at ffffffffa05f43b9 [ptlrpc]
 #7 [ffff880120b2bbe0] cfs_hash_bd_from_key at ffffffffa03db192 [libcfs]
 #8 [ffff880120b2bc10] cfs_hash_bd_get at ffffffffa03db8a5 [libcfs]
 #9 [ffff880120b2bc20] ldlm_resource_get at ffffffffa05f7099 [ptlrpc]
#10 [ffff880120b2bc90] ldlm_resource_iterate at ffffffffa0609b1d [ptlrpc]
#11 [ffff880120b2bcc0] mdc_find_cbdata at ffffffffa08a63cc [mdc]
#12 [ffff880120b2bd20] lmv_find_cbdata at ffffffffa0b17501 [lmv]
#13 [ffff880120b2bd80] find_cbdata at ffffffffa09ecdd2 [lustre]
#14 [ffff880120b2bdb0] ll_d_iput at ffffffffa09ed8be [lustre]
#15 [ffff880120b2bdd0] shrink_dcache_for_umount_subtree at ffffffff8117b36d
#16 [ffff880120b2be20] shrink_dcache_for_umount at ffffffff8117b4f6
#17 [ffff880120b2be40] generic_shutdown_super at ffffffff811653ef
#18 [ffff880120b2be60] kill_anon_super at ffffffff81165546
#19 [ffff880120b2be80] lustre_kill_super at ffffffffa04c699a [obdclass]
#20 [ffff880120b2bea0] deactivate_super at ffffffff811664e0
#21 [ffff880120b2bec0] mntput_no_expire at ffffffff811826bf
#22 [ffff880120b2bef0] sys_umount at ffffffff81183188
#23 [ffff880120b2bf80] system_call_fastpath at ffffffff810030f2
    RIP: 00007fd4f64ccd67  RSP: 00007fff034b8a38  RFLAGS: 00010202
    RAX: 00000000000000a6  RBX: ffffffff810030f2  RCX: 0000000000000010
    RDX: 0000000000000000  RSI: 0000000000000000  RDI: 00007fd4f82f8f80
    RBP: 00007fd4f82f8f40   R8: 00007fd4f82f8fa0   R9: 0000000000000000
    R10: 00007fff034b8860  R11: 0000000000000246  R12: 0000000000000000
    R13: 0000000000000000  R14: 0000000000000000  R15: 00007fd4f82f8fe0
    ORIG_RAX: 00000000000000a6  CS: 0033  SS: 002b

#14 [ffff880120b2bdb0] ll_d_iput at ffffffffa09ed8be [lustre]
    ffff880120b2bdb8: 0000000000000246 ffff880593ec4600                <-- ici ffff880593ec4600 = rbx
    ffff880120b2bdc8: ffff880120b2be18 ffffffff8117b36d
#15 [ffff880120b2bdd0] shrink_dcache_for_umount_subtree at ffffffff8117b36d

crash> struct dentry ffff880593ec4600
struct dentry {
  d_count = {
    counter = 0
  },
  d_flags = 24,
  d_lock = {
    raw_lock = {
      slock = 327685
    }
  },
  d_mounted = 1768293427,
  d_inode = 0x0,
  d_hash = {
    next = 0x0,
    pprev = 0xdead000000200200
  },
  d_parent = 0xffff88053dd10180,
  d_name = {
    hash = 302013360,
    len = 25,
    name = 0xffff880593ec46a0 "[0x2000abc55:0x13886:0x0]"
  },
  d_lru = {
    next = 0xffff880593ec4640,
    prev = 0xffff880593ec4640
  },
  d_u = {
    d_child = {
      next = 0xdead000000100100,   <-- LIST_POISON1
      prev = 0xdead000000200200    <-- LIST_POISON2
    },
    d_rcu = {
      next = 0xdead000000100100,
      func = 0xdead000000200200
    }
  },
  d_subdirs = {
    next = 0xffff880593ec4660,
    prev = 0xffff880593ec4660
  },
  d_alias = {
    next = 0xffff880593ec4670,
    prev = 0xffff880593ec4670
  },
  d_time = 18446612139194222114,
  d_op = 0xffffffffa0a64140,
  d_sb = 0xffff88033cd03000,
  d_fsdata = 0xffff8804ed89bf40,
  d_iname = "[0x2000abc55:0x13886:0x0]\000\000\t®\033&{"
}

The two dentry have the same d_parent, so it's possible to say 
that the function shrink_dcache_for_umount_subtree() have finish 
its jobs on the dentry ffff8805d305a180 that ll_drop_dentry() 
try to remove/drop.

crash> struct dentry.d_parent ffff880593ec4600
  d_parent = 0xffff88053dd10180,
crash> struct dentry.d_parent ffff8805d305a180
  d_parent = 0xffff88053dd10180,
crash> struct dentry.d_subdirs 0xffff88053dd10180
  d_subdirs = {
    next = 0xffff880356ad4c10,
    prev = 0xffff880600534190
  },

CPU7 run thread ldlm_bl_44 (pid 26203) ll_drop_dentry(0xffff8805d305a180) 
"lustre/llite/dcache.c" 267 int ll_drop_dentry(0xffff8805d305a180)
                         280                 dput(0xffff8805d305a180);

CPU0 run thread umount (pid 10428) ll_d_iput(0xffff880593ec4600)
"fs/dcache.c"           626 static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
                         695                     dentry->d_op->d_iput(0xffff880593ec4600, inode);


