:::::::::::::: mdt_reconstruct_open/mdt_reconstruct_open.txt.1 :::::::::::::: 2013-03-06 16:00:01 Lustre: DEBUG MARKER: Wed Mar 6 16:00:01 2013 2013-03-06 16:00:01 2013-03-06 16:00:37 LDISKFS-fs warning (device dm-4): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:00:37 LDISKFS-fs warning (device dm-4): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 16:00:37 2013-03-06 16:00:37 LDISKFS-fs warning (device dm-6): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:00:37 LDISKFS-fs warning (device dm-6): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 16:00:37 2013-03-06 16:01:04 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:01:04 LustreError: Skipped 127 previous similar messages 2013-03-06 16:01:19 LDISKFS-fs (dm-4): barriers disabled 2013-03-06 16:01:19 LDISKFS-fs (dm-6): barriers disabled 2013-03-06 16:01:21 LDISKFS-fs (dm-6): 35 orphan inodes deleted 2013-03-06 16:01:21 LDISKFS-fs (dm-6): recovery complete 2013-03-06 16:01:21 LDISKFS-fs (dm-6): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:01:21 LDISKFS-fs warning (device dm-6): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:01:21 LDISKFS-fs (dm-6): barriers disabled 2013-03-06 16:01:21 LDISKFS-fs (dm-4): 14 orphan inodes deleted 2013-03-06 16:01:21 LDISKFS-fs (dm-4): recovery complete 2013-03-06 16:01:21 LDISKFS-fs (dm-4): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:01:21 LDISKFS-fs warning (device dm-4): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:01:21 LDISKFS-fs (dm-6): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:01:21 Lustre: Enabling ACL 2013-03-06 16:01:21 Lustre: Enabling user_xattr 2013-03-06 16:01:21 LDISKFS-fs (dm-4): barriers disabled 2013-03-06 16:01:22 LustreError: 137-5: ptmp2-MDT0000: Not available for connect from JO.BOB.IL.LLF@o2ib11 (not set up) 2013-03-06 16:01:22 LustreError: Skipped 1 previous similar message 2013-03-06 16:01:22 LustreError: 137-5: ptmp2-MDT0000: Not available for connect from JO.BOB.IL.BAB@o2ib11 (not set up) 2013-03-06 16:01:22 LustreError: Skipped 24 previous similar messages 2013-03-06 16:01:22 LustreError: 137-5: ptmp2-MDT0000: Not available for connect from JO.BOB.IL.LIA@o2ib11 (not set up) 2013-03-06 16:01:22 LDISKFS-fs (dm-4): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:01:22 LustreError: Skipped 203 previous similar messages 2013-03-06 16:01:23 LustreError: 137-5: ptmp2-MDT0000: Not available for connect from JO.BOB.PW.PA@o2ib11 (not set up) 2013-03-06 16:01:23 LustreError: Skipped 38 previous similar messages 2013-03-06 16:01:24 Lustre: mdd_obd-ptmp2-MDT0000: changelog on 2013-03-06 16:01:25 Lustre: ptmp2-MDT0000: used disk, loading 2013-03-06 16:01:25 Lustre: 6851:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service ptmp2-MDT0000, 3394 recoverable clients, last_transno 128854337378 2013-03-06 16:01:26 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.WZ.PL@o2ib11 2013-03-06 16:01:26 Lustre: Skipped 8 previous similar messages 2013-03-06 16:01:42 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.II.LZ@o2ib11 2013-03-06 16:01:42 Lustre: Skipped 358 previous similar messages 2013-03-06 16:01:42 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:01:42 LustreError: Skipped 5716 previous similar messages 2013-03-06 16:02:11 Lustre: Enabling ACL 2013-03-06 16:02:11 Lustre: Enabling user_xattr 2013-03-06 16:02:12 LustreError: 137-5: spool2-MDT0000: Not available for connect from JO.BOO.IL.LLF@o2ib10 (not set up) 2013-03-06 16:02:12 Lustre: 7238:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 22d66de5-8fb1-b8b3-e7e0-153aea8e7c7f@JO.BOB.IL.LLF@o2ib11 recovering/t0 exp ffff88031a69b000 cur 1362582132 last 1362582085 2013-03-06 16:02:12 Lustre: 7238:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 405 previous similar messages 2013-03-06 16:02:12 LustreError: Skipped 411 previous similar messages 2013-03-06 16:02:12 Lustre: spool2-MDT0000: used disk, loading 2013-03-06 16:02:12 Lustre: 7850:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service spool2-MDT0000, 3396 recoverable clients, last_transno 120259205266 2013-03-06 16:02:14 LustreError: 7866:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b459fcc00 x1428763175053941/t0(120259163664) o101->11de0642-2911-f0b7-6d42-856c956f866b@JO.BOO.PL.LBW@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582151 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:02:16 LustreError: 7869:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880f11ba1000 x1428756067269693/t0(120259201812) o101->0db72768-cf0c-8431-63f7-55c548b93204@JO.BOO.WT.FO@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582153 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:02:16 LustreError: 7869:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 10 previous similar messages 2013-03-06 16:02:16 LustreError: 27965:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880f1175e800 x1428756056762598/t0(120259192429) o101->fe52e28e-cd2a-dd64-e607-1cd1db876622@JO.BOO.WT.AT@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582153 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:02:16 LustreError: 27965:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 6 previous similar messages 2013-03-06 16:02:30 Lustre: 8120:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from d5ab7790-d9b7-d8af-29b2-d264db7ae029@JO.BOB.II.LP@o2ib11 recovering/t0 exp ffff88031a773c00 cur 1362582150 last 1362582085 2013-03-06 16:02:30 Lustre: 8120:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 3452 previous similar messages 2013-03-06 16:02:38 LustreError: 28035:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b42bd3400 x1428763208624765/t0(120259174089) o101->e6b01462-1c46-402a-213f-62ab23788779@JO.BOO.PW.PT@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582200 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:02:38 LustreError: 28035:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 8 previous similar messages 2013-03-06 16:03:09 Lustre: 7895:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 991433e1-a58a-b452-c6bb-f63334a1d400@JO.BOO.IO.IL@o2ib10 recovering/t0 exp ffff880f1cd40000 cur 1362582189 last 1362582132 2013-03-06 16:03:09 Lustre: 7895:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 753 previous similar messages 2013-03-06 16:03:11 LustreError: 7894:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b38686800 x1428762972802599/t0(120259197202) o101->4f0fab6e-fdcf-0be8-f52d-31ca4d3fd41b@JO.BOO.PO.LA@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582233 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:03:11 LustreError: 7894:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 213 previous similar messages 2013-03-06 16:03:16 LustreError: 8121:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b3740e400 x1428749773065898/t0(120259196886) o101->5b573363-9839-3ce8-0e19-3eda694197f0@JO.BOO.WA.WF@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582238 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:03:16 LustreError: 8121:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 67 previous similar messages 2013-03-06 16:03:27 LustreError: 27897:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b36a89c00 x1428762037510634/t0(120259189910) o101->f768b0b3-3503-94a6-51cc-1ce1c36f3970@JO.BOO.IL.ZF@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362582249 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:03:27 LustreError: 27897:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 11 previous similar messages 2013-03-06 16:03:34 Lustre: MGS: haven't heard from client 09f4a691-3782-02e0-5555-7e42a90b6699 (at JO.BOO.WL.IL@o2ib10) in 227 seconds. I think it's dead, and I am evicting it. exp ffff881057f1ec00, cur 1362582214 expire 1362582064 last 1362581987 2013-03-06 16:04:59 Lustre: 6042:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from a09f9356-3b98-0224-b0f9-c79774ebfb76@JO.BOO.WL.BLA@o2ib10 recovering/t120259084904 exp ffff880f1f043800 cur 1362582299 last 1362582132 2013-03-06 16:04:59 Lustre: 6042:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 2570 previous similar messages 2013-03-06 16:05:00 Lustre: ptmp2-MDT0000: sending delayed replies to recovered clients 2013-03-06 16:05:01 LustreError: 31751:0:(mdt_open.c:1023:mdt_reconstruct_open()) ASSERTION( (!(rc < 0) || (lustre_msg_get_transno(req->rq_repmsg) == 0)) ) failed: 2013-03-06 16:05:01 LustreError: 31751:0:(mdt_open.c:1023:mdt_reconstruct_open()) LBUG 2013-03-06 16:05:01 Pid: 31751, comm: mdt_145 2013-03-06 16:05:01 2013-03-06 16:05:01 Call Trace: 2013-03-06 16:05:01 [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] 2013-03-06 16:05:01 [] lbug_with_loc+0x47/0xb0 [libcfs] 2013-03-06 16:05:01 [] mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 16:05:01 [] mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 16:05:01 [] mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 16:05:01 [] mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 16:05:01 [] mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 16:05:01 [] ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 16:05:01 [] ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 16:05:01 [] mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 16:05:01 [] mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 16:05:01 [] mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 16:05:01 [] ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? __switch_to+0x1ac/0x320 2013-03-06 16:05:01 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] child_rip+0xa/0x20 2013-03-06 16:05:01 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? child_rip+0x0/0x20 2013-03-06 16:05:01 2013-03-06 16:05:01 Kernel panic - not syncing: LBUG 2013-03-06 16:05:01 Pid: 31751, comm: mdt_145 Not tainted 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 #1 2013-03-06 16:05:01 Call Trace: 2013-03-06 16:05:01 [] ? panic+0x78/0x143 2013-03-06 16:05:01 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] 2013-03-06 16:05:01 [] ? mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 16:05:01 [] ? mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 16:05:01 [] ? mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 16:05:01 [] ? mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 16:05:01 [] ? mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 16:05:01 [] ? ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 16:05:01 [] ? ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 16:05:01 [] ? mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 16:05:01 [] ? mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 16:05:01 [] ? mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 16:05:01 [] ? ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? __switch_to+0x1ac/0x320 2013-03-06 16:05:01 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? child_rip+0xa/0x20 2013-03-06 16:05:01 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:05:01 [] ? child_rip+0x0/0x20 2013-03-06 16:05:01 Initializing cgroup subsys cpuset 2013-03-06 16:05:01 Initializing cgroup subsys cpu 2013-03-06 16:05:01 Linux version 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 (efix@atlas.frec.bull.fr) (gcc version 4.4.5 20110214 (Bull 4.4.5-6) (GCC) ) #1 SMP Thu Jul 5 16:46:35 CEST 2012 :::::::::::::: mdt_reconstruct_open/mdt_reconstruct_open.txt.2 :::::::::::::: 2013-03-06 16:16:12 lascaux110 login: Lustre: Lustre: Build Version: T-2_1_4_0-lustrebull-EFIX_319613_AE2_2-20130227105145-CHANGED-2.6.32-220.23.1.bl6.Bull.28.8.x86_64 2013-03-06 16:58:09 LustreError: 16260:0:(o2iblnd_cb.c:2248:kiblnd_passive_connect()) Can't accept JO.BOO.WZ.AA@o2ib10 on NA (ib0:0:JO.BOO.WL.IB): bad dst nid JO.BOO.WL.IB@o2ib10 2013-03-06 16:58:09 LustreError: 16244:0:(o2iblnd_cb.c:2248:kiblnd_passive_connect()) Can't accept JO.BOO.IW.WP@o2ib10 on NA (ib0:0:JO.BOO.WL.IB): bad dst nid JO.BOO.WL.IB@o2ib10 2013-03-06 16:58:09 Lustre: Added LNI JO.BOO.WL.IB@o2ib10 [8/64/0/180] 2013-03-06 16:58:09 LustreError: 16244:0:(o2iblnd_cb.c:2248:kiblnd_passive_connect()) Skipped 2 previous similar messages 2013-03-06 16:58:09 Lustre: Added LNI JO.BOB.WL.IB@o2ib11 [8/64/0/180] 2013-03-06 16:58:09 LDISKFS-fs warning (device dm-1): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:10 LDISKFS-fs (dm-1): barriers disabled 2013-03-06 16:58:10 LDISKFS-fs warning (device dm-3): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:10 LDISKFS-fs (dm-3): barriers disabled 2013-03-06 16:58:10 LDISKFS-fs (dm-1): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:10 LDISKFS-fs warning (device dm-1): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:10 LDISKFS-fs (dm-1): barriers disabled 2013-03-06 16:58:11 LDISKFS-fs (dm-3): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:11 LDISKFS-fs warning (device dm-3): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:11 LDISKFS-fs (dm-3): barriers disabled 2013-03-06 16:58:11 Lustre: Lustre OSC module (ffffffffa0a62100). 2013-03-06 16:58:11 Lustre: Lustre LOV module (ffffffffa0af5ce0). 2013-03-06 16:58:11 Lustre: Lustre client module (ffffffffa0be0960). 2013-03-06 16:58:11 LDISKFS-fs (dm-1): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:11 LDISKFS-fs warning (device dm-7): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:11 LDISKFS-fs warning (device dm-7): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 16:58:11 2013-03-06 16:58:12 LDISKFS-fs (dm-3): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:12 LustreError: 31911:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff8803f1cdc400 x1428774441713667/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1 2013-03-06 16:58:12 LustreError: 31953:0:(ldlm_resource.c:751:ldlm_resource_complain()) Namespace MGCJO.BOO.WL.IB@o2ib10 resource refcount nonzero (1) after lock cleanup; forcing cleanup. 2013-03-06 16:58:12 LustreError: 31953:0:(ldlm_resource.c:757:ldlm_resource_complain()) Resource: ffff88044c563e40 (3632262450976547699/0/0/0) (rc: 1) 2013-03-06 16:58:12 LustreError: 31911:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff8803f1cdc400 x1428774441713668/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1 2013-03-06 16:58:12 Lustre: MGCJO.BOO.WL.IB@o2ib10: Reactivating import 2013-03-06 16:58:12 Lustre: Enabling ACL 2013-03-06 16:58:12 Lustre: Enabling user_xattr 2013-03-06 16:58:14 Lustre: mdd_obd-scratch2-MDT0000: changelog on 2013-03-06 16:58:14 Lustre: scratch2-MDT0000: used disk, loading 2013-03-06 16:58:14 Lustre: 32529:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service scratch2-MDT0000, 3394 recoverable clients, last_transno 94492245062 2013-03-06 16:58:14 LustreError: 137-5: scratch2-MDT0000: Not available for connect from JO.BOB.WA.IO@o2ib11 (not set up) 2013-03-06 16:58:14 LustreError: 137-5: UUID 'ptmp2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:14 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:14 Lustre: scratch2-MDT0000: Temporarily refusing client connection from JO.BOB.PO.AO@o2ib11 2013-03-06 16:58:14 LustreError: Skipped 1 previous similar message 2013-03-06 16:58:14 Lustre: scratch2-MDT0000: Temporarily refusing client connection from JO.BOB.PB.PI@o2ib11 2013-03-06 16:58:14 Lustre: Skipped 1 previous similar message 2013-03-06 16:58:15 Lustre: scratch2-MDT0000: Temporarily refusing client connection from JO.BOB.PI.FF@o2ib11 2013-03-06 16:58:15 LustreError: 137-5: UUID 'ptmp2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:15 LustreError: Skipped 20 previous similar messages 2013-03-06 16:58:15 Lustre: Skipped 9 previous similar messages 2013-03-06 16:58:16 Lustre: scratch2-MDT0000: Temporarily refusing client connection from JO.BOB.WI.LLT@o2ib11 2013-03-06 16:58:16 LustreError: 137-5: UUID 'ptmp2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:16 LustreError: Skipped 43 previous similar messages 2013-03-06 16:58:16 Lustre: Skipped 21 previous similar messages 2013-03-06 16:58:18 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:18 LustreError: Skipped 602 previous similar messages 2013-03-06 16:58:18 Lustre: scratch2-MDT0000: Temporarily refusing client connection from JO.BOB.PO.LIB@o2ib11 2013-03-06 16:58:18 Lustre: Skipped 142 previous similar messages 2013-03-06 16:58:20 Lustre: 32544:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from 914acd7a-9ce1-ce57-878e-09913413e322@JO.BOB.WP.TL@o2ib11 recovering/t0 exp ffff880861938c00 cur 1362585500 last 1362585494 2013-03-06 16:58:20 Lustre: 32544:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from 4d96d940-9eb4-5543-915e-b5735af6a056@JO.BOB.WI.PI@o2ib11 recovering/t0 exp ffff8808618d6c00 cur 1362585500 last 1362585494 2013-03-06 16:58:21 Lustre: 32604:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from 98d7d17a-4f75-44dc-1b6d-574fc53d479a@JO.BOB.IZ.AB@o2ib11 recovering/t0 exp ffff88086c938800 cur 1362585501 last 1362585494 2013-03-06 16:58:21 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880ecddaa800 x1428774441713671/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:21 Lustre: 32604:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 100 previous similar messages 2013-03-06 16:58:21 Lustre: 32014:0:(client.c:1833:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1362585492/real 1362585492] req@ffff880bec10a800 x1428774441713669/t0(0) o250->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 368/512 e 0 to 1 dl 1362585497 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 2013-03-06 16:58:22 Lustre: 32542:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from 21de1b74-daa9-7dd5-afb6-4ecf2ee432a5@JO.BOB.WA.WT@o2ib11 recovering/t0 exp ffff880861a08400 cur 1362585502 last 1362585494 2013-03-06 16:58:22 Lustre: 32542:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 211 previous similar messages 2013-03-06 16:58:22 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:22 LustreError: Skipped 2299 previous similar messages 2013-03-06 16:58:24 Lustre: 32613:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from 372ff9a6-5de0-460e-3b10-3430a41f6818@JO.BOB.II.BPO@o2ib11 recovering/t0 exp ffff880861ed3800 cur 1362585504 last 1362585494 2013-03-06 16:58:24 Lustre: 32613:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 163 previous similar messages 2013-03-06 16:58:27 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880ecddaa800 x1428774441713687/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:28 Lustre: 32598:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from 4432079c-650d-c673-1346-997dcc6caa74@JO.BOB.II.BOW@o2ib11 recovering/t0 exp ffff880861ee2400 cur 1362585508 last 1362585494 2013-03-06 16:58:28 Lustre: 32598:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 64 previous similar messages 2013-03-06 16:58:30 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:30 LustreError: Skipped 1217 previous similar messages 2013-03-06 16:58:31 LustreError: 31953:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880b6d28a800 x1428774441713688/t0(0) o253->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 4736/4736 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:33 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880ecddaa800 x1428774441713689/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:37 LustreError: 31953:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880b6d28a800 x1428774441713690/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:39 Lustre: 32544:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from cb112178-63b5-48fb-5f59-6499d6d80021@JO.BOB.PO.AO@o2ib11 recovering/t0 exp ffff88086c9e7800 cur 1362585519 last 1362585494 2013-03-06 16:58:39 Lustre: 32544:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 338 previous similar messages 2013-03-06 16:58:43 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880ee8b56000 x1428774441713691/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:43 Lustre: Enabling ACL 2013-03-06 16:58:43 Lustre: Enabling user_xattr 2013-03-06 16:58:43 LustreError: 137-5: cprot2-MDT0000: Not available for connect from JO.BOO.PO.BZZ@o2ib10 (not set up) 2013-03-06 16:58:44 Lustre: cprot2-MDT0000: used disk, loading 2013-03-06 16:58:44 LustreError: 137-5: cprot2-MDT0000: Not available for connect from JO.BOO.PO.BFZ@o2ib10 (not set up) 2013-03-06 16:58:44 LustreError: Skipped 64 previous similar messages 2013-03-06 16:58:44 Lustre: 576:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service cprot2-MDT0000, 3394 recoverable clients, last_transno 98785233635 2013-03-06 16:58:44 Lustre: cprot2-MDT0000: Temporarily refusing client connection from JO.BOO.PO.LOP@o2ib10 2013-03-06 16:58:44 Lustre: Skipped 209 previous similar messages 2013-03-06 16:58:46 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:46 LustreError: Skipped 1647 previous similar messages 2013-03-06 16:58:52 Lustre: cprot2-MDT0000: Temporarily refusing client connection from JO.BOO.IP.AT@o2ib10 2013-03-06 16:58:52 Lustre: Skipped 262 previous similar messages 2013-03-06 16:58:53 LDISKFS-fs (dm-7): barriers disabled 2013-03-06 16:58:54 LDISKFS-fs (dm-7): recovery complete 2013-03-06 16:58:54 LDISKFS-fs (dm-7): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:54 LDISKFS-fs warning (device dm-7): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:54 LDISKFS-fs (dm-7): barriers disabled 2013-03-06 16:58:54 LDISKFS-fs (dm-7): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:54 Lustre: MGS MGS started 2013-03-06 16:58:54 Lustre: MGCJO.BOO.WL.IB@o2ib10: Reactivating import 2013-03-06 16:58:55 Lustre: 32579:0:(ldlm_lib.c:952:target_handle_connect()) scratch2-MDT0000: connection from a2911316-a2e6-32fa-37af-0a1851cbb2b4@JO.BOB.IL.BPL@o2ib11 recovering/t94489304208 exp ffff880861d95400 cur 1362585535 last 1362585494 2013-03-06 16:58:55 Lustre: 32579:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1420 previous similar messages 2013-03-06 16:59:05 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880ec4850400 x1428774441713918/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:59:05 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) Skipped 2 previous similar messages 2013-03-06 16:59:11 LustreError: 646:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880ed7647c00 x1425512593047985/t0(90288826703) o101->65618f78-8b5e-95fa-026b-de58a53bf307@JO.BOO.WZ.BBB@o2ib10:0/0 lens 632/4936 e 0 to 0 dl 1362585592 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:59:11 LustreError: 648:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880ed7681400 x1425512146008687/t0(90285581556) o101->c25aa139-5718-55c0-d2e3-7d3f36552a0f@JO.BOO.PI.W@o2ib10:0/0 lens 568/4936 e 0 to 0 dl 1362585592 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:59:14 Lustre: 32014:0:(client.c:1833:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1362585534/real 1362585534] req@ffff88083bbbc000 x1428774441713845/t0(0) o250->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 368/512 e 0 to 1 dl 1362585554 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 2013-03-06 16:59:18 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:59:18 LustreError: Skipped 1870 previous similar messages 2013-03-06 16:59:23 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880ec4850400 x1428774441713941/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IL@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:59:23 LustreError: 32528:0:(client.c:1065:ptlrpc_import_delay_req()) Skipped 3 previous similar messages 2013-03-06 16:59:25 Lustre: MGCJO.BOO.WL.IB@o2ib10: Reactivating import 2013-03-06 16:59:27 Lustre: 836:0:(ldlm_lib.c:952:target_handle_connect()) MGS: connection from e67f2ab9-0c68-8920-516c-9ece5754ea65@JO.BOO.IO.F@o2ib10 t0 exp (null) cur 1362585567 last 0 2013-03-06 16:59:27 Lustre: 836:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 3766 previous similar messages Console [lascaux110] log at 2013-03-06 17:00:00 CET. 2013-03-06 17:00:01 Lustre: DEBUG MARKER: Wed Mar 6 17:00:01 2013 2013-03-06 17:00:01 2013-03-06 17:00:39 Lustre: 1273:0:(ldlm_lib.c:952:target_handle_connect()) MGS: connection from dc541868-9bfb-295f-aabd-025fafd085b9@JO.BOO.WZ.BAP@o2ib10 t0 exp (null) cur 1362585639 last 0 2013-03-06 17:00:39 Lustre: 1273:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1737 previous similar messages 2013-03-06 17:03:20 Lustre: scratch2-MDT0000: recovery is timed out, evict stale exports 2013-03-06 17:03:20 Lustre: scratch2-MDT0000: disconnecting 16 stale clients 2013-03-06 17:03:21 Lustre: scratch2-MDT0000: sending delayed replies to recovered clients 2013-03-06 17:03:21 Lustre: MDS mdd_obd-scratch2-MDT0000: scratch2-OST0001_UUID now active, resetting orphans 2013-03-06 17:03:21 Lustre: Skipped 3 previous similar messages 2013-03-06 17:03:21 Lustre: MDS mdd_obd-scratch2-MDT0000: scratch2-OST0009_UUID now active, resetting orphans 2013-03-06 17:03:21 Lustre: Skipped 2 previous similar messages 2013-03-06 17:03:22 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:23 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:23 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:23 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 1 previous similar message 2013-03-06 17:03:25 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:25 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 3 previous similar messages 2013-03-06 17:03:27 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:27 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 5 previous similar messages 2013-03-06 17:03:31 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:31 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 11 previous similar messages 2013-03-06 17:03:39 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:39 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 18 previous similar messages 2013-03-06 17:03:56 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 17:03:56 Lustre: 32541:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 31 previous similar messages 2013-03-06 17:04:04 Lustre: cprot2-MDT0000: recovery is timed out, evict stale exports 2013-03-06 17:04:04 Lustre: cprot2-MDT0000: disconnecting 16 stale clients 2013-03-06 17:04:05 Lustre: cprot2-MDT0000: sending delayed replies to recovered clients 2013-03-06 17:04:05 Lustre: MDS mdd_obd-cprot2-MDT0000: cprot2-OST0066_UUID now active, resetting orphans 2013-03-06 17:04:05 Lustre: Skipped 67 previous similar messages 2013-03-06 17:04:15 Lustre: 1276:0:(ldlm_lib.c:952:target_handle_connect()) MGS: connection from a20bc352-ce85-d233-5232-35ebae30096f@JO.BOO.WL.BWL@o2ib10 t0 exp (null) cur 1362585855 last 0 2013-03-06 17:04:15 Lustre: 1276:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 2352 previous similar messages 2013-03-06 17:04:32 LDISKFS-fs warning (device dm-4): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 17:04:32 LDISKFS-fs warning (device dm-4): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 17:04:32 2013-03-06 17:04:32 LDISKFS-fs warning (device dm-5): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 17:04:32 LDISKFS-fs warning (device dm-5): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 17:04:32 2013-03-06 17:04:59 LustreError: 137-5: UUID 'ptmp2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 17:04:59 LustreError: Skipped 2282 previous similar messages 2013-03-06 17:05:01 Lustre: DEBUG MARKER: Wed Mar 6 17:05:01 2013 2013-03-06 17:05:01 2013-03-06 17:05:14 LDISKFS-fs (dm-4): barriers disabled 2013-03-06 17:05:14 LDISKFS-fs (dm-5): barriers disabled 2013-03-06 17:05:15 LDISKFS-fs (dm-5): recovery complete 2013-03-06 17:05:15 LDISKFS-fs (dm-5): mounted filesystem with ordered data mode. Opts: 2013-03-06 17:05:15 LDISKFS-fs warning (device dm-5): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 17:05:15 LDISKFS-fs (dm-5): barriers disabled 2013-03-06 17:05:15 LDISKFS-fs (dm-4): recovery complete 2013-03-06 17:05:15 LDISKFS-fs (dm-4): mounted filesystem with ordered data mode. Opts: 2013-03-06 17:05:15 LDISKFS-fs warning (device dm-4): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 17:05:15 LDISKFS-fs (dm-4): barriers disabled 2013-03-06 17:05:16 LDISKFS-fs (dm-5): mounted filesystem with ordered data mode. Opts: 2013-03-06 17:05:16 Lustre: Enabling ACL 2013-03-06 17:05:16 Lustre: Enabling user_xattr 2013-03-06 17:05:16 LDISKFS-fs (dm-4): mounted filesystem with ordered data mode. Opts: 2013-03-06 17:05:18 Lustre: mdd_obd-ptmp2-MDT0000: changelog on 2013-03-06 17:05:18 Lustre: ptmp2-MDT0000: used disk, loading 2013-03-06 17:05:18 Lustre: 4650:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service ptmp2-MDT0000, 3378 recoverable clients, last_transno 137438956898 2013-03-06 17:05:24 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.IA.F@o2ib11 2013-03-06 17:05:24 Lustre: Skipped 308 previous similar messages 2013-03-06 17:05:56 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.II.WF@o2ib11 2013-03-06 17:05:56 Lustre: Skipped 5102 previous similar messages 2013-03-06 17:06:05 LustreError: 2841:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff8808675ae050 x1428756046290681/t0(128854126694) o101->e1d2c582-9e75-1a6c-8e54-f4337f7ca662@JO.BOB.WZ.PW@o2ib11:0/0 lens 552/4936 e 0 to 0 dl 1362586007 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:06:05 Lustre: Enabling ACL 2013-03-06 17:06:05 Lustre: Enabling user_xattr 2013-03-06 17:06:06 LustreError: 137-5: spool2-MDT0000: Not available for connect from JO.BOO.WZ.BBZ@o2ib10 (not set up) 2013-03-06 17:06:06 LustreError: Skipped 31 previous similar messages 2013-03-06 17:06:06 Lustre: spool2-MDT0000: used disk, loading 2013-03-06 17:06:06 Lustre: 4972:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service spool2-MDT0000, 3396 recoverable clients, last_transno 120259205266 Console [lascaux110] joined by on pts/63 at 03-06 17:06. 2013-03-06 17:06:12 2013-03-06 17:06:12 bullx Linux Server release 6.1 (V1) 2013-03-06 17:06:12 Kernel 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 on an x86_64 2013-03-06 17:06:12 2013-03-06 17:06:12 lascaux110 login: Console [lascaux110] departed by on pts/63 at 03-06 17:06. 2013-03-06 17:06:30 LustreError: 5235:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b5ace9000 x1428756071456930/t0(128854126900) o101->f770fcca-c4c5-8d1b-a30d-4330bc04747d@JO.BOB.WZ.PB@o2ib11:0/0 lens 552/4936 e 0 to 0 dl 1362586031 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:06:32 LustreError: 5253:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880ed0950000 x1428763254753049/t0(120259170867) o101->78fd490f-5fae-5357-4ddf-25884e216e00@JO.BOO.PW.IP@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362586033 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:06:32 LustreError: 5253:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 2 previous similar messages 2013-03-06 17:06:50 LustreError: 3008:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880ec64ea850 x1428762205289484/t0(120259197459) o101->0c9a6153-aaba-0362-bf87-aa63103427d7@JO.BOO.II.LBZ@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362586052 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:06:50 LustreError: 3008:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 43 previous similar messages 2013-03-06 17:06:55 sd 9:0:0:3: [sdq] Warning! Received an indication that the LUN assignments on this target have changed. The Linux SCSI layer does not automatically remap LUN assignments. 2013-03-06 17:06:55 sd 8:0:0:4: [sdn] Warning! Received an indication that the LUN assignments on this target have changed. The Linux SCSI layer does not automatically remap LUN assignments. 2013-03-06 17:06:55 sd 7:0:0:2: [sdh] Warning! Received an indication that the LUN assignments on this target have changed. The Linux SCSI layer does not automatically remap LUN assignments. 2013-03-06 17:06:58 sd 6:0:0:2: [sdd] Warning! Received an indication that the LUN assignments on this target have changed. The Linux SCSI layer does not automatically remap LUN assignments. 2013-03-06 17:07:29 Lustre: MGS: haven't heard from client e99b62f5-62dd-97d4-f461-73c99f628983 (at JO.BOO.WL.IL@o2ib10) in 227 seconds. I think it's dead, and I am evicting it. exp ffff880b6f7f4400, cur 1362586049 expire 1362585899 last 1362585822 2013-03-06 17:07:34 LustreError: 6404:0:(mdt_open.c:1023:mdt_reconstruct_open()) ASSERTION( (!(rc < 0) || (lustre_msg_get_transno(req->rq_repmsg) == 0)) ) failed: 2013-03-06 17:07:34 LustreError: 6404:0:(mdt_open.c:1023:mdt_reconstruct_open()) LBUG 2013-03-06 17:07:34 Pid: 6404, comm: mdt_124 2013-03-06 17:07:34 2013-03-06 17:07:34 Call Trace: 2013-03-06 17:07:34 [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] 2013-03-06 17:07:34 [] lbug_with_loc+0x47/0xb0 [libcfs] 2013-03-06 17:07:34 [] mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 17:07:34 [] mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 17:07:34 [] mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 17:07:34 [] mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 17:07:34 [] mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 17:07:34 [] ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 17:07:34 [] ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 17:07:34 [] mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 17:07:34 [] mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 17:07:34 [] mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 17:07:34 [] ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? fput+0x25/0x30 2013-03-06 17:07:34 [] ? __switch_to+0x1ac/0x320 2013-03-06 17:07:34 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] child_rip+0xa/0x20 2013-03-06 17:07:34 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? child_rip+0x0/0x20 2013-03-06 17:07:34 2013-03-06 17:07:34 Kernel panic - not syncing: LBUG 2013-03-06 17:07:34 Pid: 6404, comm: mdt_124 Not tainted 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 #1 2013-03-06 17:07:34 Call Trace: 2013-03-06 17:07:34 [] ? panic+0x78/0x143 2013-03-06 17:07:34 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] 2013-03-06 17:07:34 [] ? mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 17:07:34 [] ? mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 17:07:34 Lustre: ptmp2-MDT0000: sending delayed replies to recovered clients 2013-03-06 17:07:34 [] ? mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 17:07:34 [] ? mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 17:07:34 [] ? mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 17:07:34 [] ? ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 17:07:34 [] ? ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 17:07:34 [] ? mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 17:07:34 [] ? mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 17:07:34 [] ? mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 17:07:34 [] ? ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? fput+0x25/0x30 2013-03-06 17:07:34 [] ? __switch_to+0x1ac/0x320 2013-03-06 17:07:34 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? child_rip+0xa/0x20 2013-03-06 17:07:34 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:07:34 [] ? child_rip+0x0/0x20 2013-03-06 17:07:34 Initializing cgroup subsys cpuset 2013-03-06 17:07:34 Initializing cgroup subsys cpu 2013-03-06 17:07:34 Linux version 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 (efix@atlas.frec.bull.fr) (gcc version 4.4.5 20110214 (Bull 4.4.5-6) (GCC) ) #1 SMP Thu Jul 5 16:46:35 CEST 2012 2013-03-06 17:07:35 Command line: ro root=UUID=8c9a4335-4b78-413c-a624-ae2ca4ccbe04 rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=C SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us console=tty0 console=ttyS1,115200 rdloaddriver=ahci rdloaddriver=aacraid rdloaddriver=megaraid_sas selinux=0 rdblacklist=lpfc transparent_hugepage=never pciehp_disable nmi_watchdog=0 nointremap intel_iommu=off audit=0 mce=0 irqpoll maxcpus=1 reset_devices nointremap intel_iommu=off memmap=exactmap memmap=624K@4K memmap=195964K@33392K elfcorehdr=229356K memmap=4K$0K memmap=12K$628K memmap=128K$896K memmap=10240K#1520344K memmap=512K$1564156K memmap=5740K#1564668K memmap=1172K#1570408K memmap=256K#1571580K memmap=1024K#1571836K memmap=278528K$2080768K memmap=16384K$4128768K memmap=16K$4174960K memmap=8192K$4186112K :::::::::::::: mdt_reconstruct_open/mdt_reconstruct_open.txt.3 :::::::::::::: Console [lascaux111] log at 2013-03-06 09:00:00 CET. 2013-03-06 09:12:38 Lustre: Lustre: Build Version: T-2_1_4_0-lustrebull-EFIX_319613_AE2_2-20130227105145-CHANGED-2.6.32-220.23.1.bl6.Bull.28.8.x86_64 2013-03-06 09:12:38 Lustre: Added LNI JO.BOO.WL.IL@o2ib10 [8/64/0/180] 2013-03-06 09:12:38 Lustre: Added LNI JO.BOB.WL.IL@o2ib11 [8/64/0/180] 2013-03-06 09:12:40 Lustre: Lustre OSC module (ffffffffa0a30100). 2013-03-06 09:12:40 Lustre: Lustre LOV module (ffffffffa0ac3ce0). 2013-03-06 09:12:40 Lustre: Lustre client module (ffffffffa0bae960). 2013-03-06 09:12:40 LDISKFS-fs warning (device dm-2): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 09:12:40 LDISKFS-fs warning (device dm-2): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 09:12:40 2013-03-06 09:13:22 LDISKFS-fs (dm-2): barriers disabled 2013-03-06 09:13:23 LDISKFS-fs (dm-2): recovery complete 2013-03-06 09:13:23 LDISKFS-fs (dm-2): mounted filesystem with ordered data mode. Opts: 2013-03-06 09:13:23 LDISKFS-fs warning (device dm-2): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 09:13:23 LDISKFS-fs (dm-2): barriers disabled 2013-03-06 09:13:23 LDISKFS-fs (dm-2): mounted filesystem with ordered data mode. Opts: 2013-03-06 09:13:23 Lustre: MGCJO.BOO.WL.IB@o2ib10: Reactivating import 2013-03-06 09:13:24 Lustre: Enabling ACL 2013-03-06 09:13:24 Lustre: Enabling user_xattr 2013-03-06 09:13:26 Lustre: mdd_obd-ptmp2-MDT0000: changelog on 2013-03-06 09:13:26 Lustre: ptmp2-MDT0000: used disk, loading 2013-03-06 09:13:26 Lustre: 22308:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service ptmp2-MDT0000, 27 recoverable clients, last_transno 125757074214 2013-03-06 09:13:28 LustreError: 11-0: an error occurred while communicating with JO.BOO.WL.BZP@o2ib10. The ost_connect operation failed with -16 2013-03-06 09:13:29 LustreError: 11-0: an error occurred while communicating with JO.BOO.WL.BTL@o2ib10. The ost_connect operation failed with -16 2013-03-06 09:13:29 LustreError: 11-0: an error occurred while communicating with JO.BOO.WL.BZI@o2ib10. The ost_connect operation failed with -16 2013-03-06 09:13:29 LustreError: Skipped 1 previous similar message 2013-03-06 09:13:31 LustreError: 11-0: an error occurred while communicating with JO.BOB.WL.BTO@o2ib11. The ost_connect operation failed with -16 2013-03-06 09:13:31 LustreError: Skipped 2 previous similar messages 2013-03-06 09:13:34 LustreError: 11-0: an error occurred while communicating with JO.BOO.WL.BZW@o2ib10. The ost_connect operation failed with -16 2013-03-06 09:13:34 LustreError: Skipped 4 previous similar messages 2013-03-06 09:13:38 LustreError: 11-0: an error occurred while communicating with JO.BOO.WL.BZA@o2ib10. The ost_connect operation failed with -16 2013-03-06 09:13:38 LustreError: Skipped 10 previous similar messages 2013-03-06 09:13:47 LustreError: 11-0: an error occurred while communicating with JO.BOO.WL.BZF@o2ib10. The ost_connect operation failed with -16 2013-03-06 09:13:47 LustreError: Skipped 12 previous similar messages 2013-03-06 09:13:54 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:13:57 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:13:58 LustreError: 137-5: UUID 'scratch2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:13:58 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.PA.BZI@o2ib11 2013-03-06 09:13:58 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.WA.BTO@o2ib11 2013-03-06 09:14:07 LustreError: 137-5: UUID 'scratch2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:14:07 LustreError: Skipped 5 previous similar messages 2013-03-06 09:14:12 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:14:13 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 07fde1da-97b5-1fd6-aab8-577ee4408de0@JO.BOB.WA.BIW@o2ib11 recovering/t0 exp ffff880c5a79b000 cur 1362557653 last 1362557606 2013-03-06 09:14:22 LustreError: 137-5: UUID 'scratch2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:14:22 Lustre: 22677:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 9cce4f70-fc14-0058-d4c9-584e892caf6a@JO.BOB.WZ.PP@o2ib11 recovering/t121733478740 exp ffff880c5a7a7800 cur 1362557662 last 1362557606 2013-03-06 09:14:23 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 8ab5f839-5485-8c94-47cc-d720c0d32321@JO.BOB.WZ.BBB@o2ib11 recovering/t124467398826 exp ffff880c5a7a4800 cur 1362557663 last 1362557606 2013-03-06 09:14:24 Lustre: 22677:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 05fbd798-a9d0-0389-3f71-3b2012f62f96@JO.BOB.WZ.BWF@o2ib11 recovering/t124266949687 exp ffff880c5a7a4400 cur 1362557664 last 1362557606 2013-03-06 09:14:24 Lustre: 22677:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 4 previous similar messages 2013-03-06 09:14:28 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 9892c88d-b586-37c8-dc86-90e5df6ffddc@JO.BOB.II.PT@o2ib11 recovering/t125551102787 exp ffff880c5a79ec00 cur 1362557668 last 1362557606 2013-03-06 09:14:28 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1 previous similar message 2013-03-06 09:14:31 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:14:31 LustreError: Skipped 13 previous similar messages 2013-03-06 09:14:40 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 59ca6d40-aace-c1f5-450a-22f61f7a23ea@JO.BOB.IZ.BAL@o2ib11 recovering/t0 exp ffff880c5a79b800 cur 1362557680 last 1362557606 2013-03-06 09:14:40 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 3 previous similar messages 2013-03-06 09:14:48 LustreError: 137-5: UUID 'scratch2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:14:48 LustreError: Skipped 7 previous similar messages 2013-03-06 09:15:16 LDISKFS-fs warning (device dm-0): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 09:15:16 LDISKFS-fs warning (device dm-0): ldiskfs_multi_mount_protect: MMP interval 42 higher than expected, please wait. 2013-03-06 09:15:16 2013-03-06 09:15:21 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:15:21 LustreError: Skipped 21 previous similar messages 2013-03-06 09:15:58 LDISKFS-fs (dm-0): barriers disabled 2013-03-06 09:15:59 LDISKFS-fs (dm-0): recovery complete 2013-03-06 09:15:59 LDISKFS-fs (dm-0): mounted filesystem with ordered data mode. Opts: 2013-03-06 09:15:59 LDISKFS-fs warning (device dm-0): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 09:15:59 LDISKFS-fs (dm-0): barriers disabled 2013-03-06 09:16:00 LDISKFS-fs (dm-0): mounted filesystem with ordered data mode. Opts: 2013-03-06 09:16:00 Lustre: Enabling ACL 2013-03-06 09:16:00 Lustre: Enabling user_xattr 2013-03-06 09:16:00 Lustre: spool2-MDT0000: used disk, loading 2013-03-06 09:16:00 Lustre: 23980:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service spool2-MDT0000, 16 recoverable clients, last_transno 115964116992 2013-03-06 09:16:01 LustreError: 11-0: an error occurred while communicating with JO.BOB.WL.BFT@o2ib11. The ost_connect operation failed with -16 2013-03-06 09:16:01 LustreError: Skipped 2 previous similar messages 2013-03-06 09:16:03 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.WZ.BBB@o2ib10 2013-03-06 09:16:03 Lustre: Skipped 2 previous similar messages 2013-03-06 09:16:04 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.PP.LP@o2ib10 2013-03-06 09:16:04 Lustre: Skipped 2 previous similar messages 2013-03-06 09:16:08 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.II.PT@o2ib10 2013-03-06 09:16:08 Lustre: Skipped 1 previous similar message 2013-03-06 09:16:18 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.WA.BIW@o2ib10 2013-03-06 09:16:18 Lustre: Skipped 3 previous similar messages 2013-03-06 09:16:27 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.WZ.PP@o2ib10 2013-03-06 09:16:27 Lustre: Skipped 2 previous similar messages 2013-03-06 09:16:28 LustreError: 137-5: UUID 'scratch2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 09:16:28 LustreError: Skipped 24 previous similar messages 2013-03-06 09:16:40 INFO: task tgt_recov:22320 blocked for more than 120 seconds. 2013-03-06 09:16:40 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 2013-03-06 09:16:40 tgt_recov D 0000000000000005 0 22320 2 0x00000000 2013-03-06 09:16:40 ffff880c5a7d3e20 0000000000000046 0000000000000000 0000000000000003 2013-03-06 09:16:40 0000000000000003 0000000000000001 ffff880c5a7d3dc0 ffffffff81042ca3 2013-03-06 09:16:40 ffff880c74f543e0 ffff880c5a7d3fd8 000000000000db00 ffff880c74f543e0 2013-03-06 09:16:40 Call Trace: 2013-03-06 09:16:40 [] ? enqueue_task+0x43/0x90 2013-03-06 09:16:40 [] ? check_for_clients+0x0/0x90 [ptlrpc] 2013-03-06 09:16:40 [] target_recovery_overseer+0x9d/0x230 [ptlrpc] 2013-03-06 09:16:40 [] ? exp_connect_healthy+0x0/0x20 [ptlrpc] 2013-03-06 09:16:40 [] ? autoremove_wake_function+0x0/0x40 2013-03-06 09:16:40 [] target_recovery_thread+0x566/0x1880 [ptlrpc] 2013-03-06 09:16:40 [] ? target_recovery_thread+0x0/0x1880 [ptlrpc] 2013-03-06 09:16:40 [] child_rip+0xa/0x20 2013-03-06 09:16:40 [] ? target_recovery_thread+0x0/0x1880 [ptlrpc] 2013-03-06 09:16:40 [] ? target_recovery_thread+0x0/0x1880 [ptlrpc] 2013-03-06 09:16:40 [] ? child_rip+0x0/0x20 2013-03-06 09:16:43 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.WA.BIW@o2ib10 2013-03-06 09:16:43 Lustre: Skipped 12 previous similar messages 2013-03-06 09:16:52 Lustre: 22321:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 0c373338-ef52-1ac4-ccef-65210923ee10@JO.BOO.WZ.PP@o2ib10 recovering/t111675073489 exp ffff880442907c00 cur 1362557812 last 1362557760 2013-03-06 09:17:08 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 57f4fc81-7b5c-7976-a52b-88eef1c2a570@JO.BOO.WA.BIW@o2ib10 recovering/t0 exp ffff8804428fb000 cur 1362557828 last 1362557760 2013-03-06 09:17:08 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 12 previous similar messages 2013-03-06 09:17:14 Lustre: spool2-MDT0000: sending delayed replies to recovered clients 2013-03-06 09:17:14 Lustre: MDS mdd_obd-spool2-MDT0000: spool2-OST0002_UUID now active, resetting orphans 2013-03-06 09:18:40 INFO: task tgt_recov:22320 blocked for more than 120 seconds. 2013-03-06 09:18:40 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 2013-03-06 09:18:40 tgt_recov D 0000000000000005 0 22320 2 0x00000000 2013-03-06 09:18:40 ffff880c5a7d3e20 0000000000000046 0000000000000000 0000000000000003 2013-03-06 09:18:40 0000000000000003 0000000000000001 ffff880c5a7d3dc0 ffffffff81042ca3 2013-03-06 09:18:40 ffff880c74f543e0 ffff880c5a7d3fd8 000000000000db00 ffff880c74f543e0 2013-03-06 09:18:40 Call Trace: 2013-03-06 09:18:40 [] ? enqueue_task+0x43/0x90 2013-03-06 09:18:40 [] ? check_for_clients+0x0/0x90 [ptlrpc] 2013-03-06 09:18:40 [] target_recovery_overseer+0x9d/0x230 [ptlrpc] 2013-03-06 09:18:40 [] ? exp_connect_healthy+0x0/0x20 [ptlrpc] 2013-03-06 09:18:40 [] ? autoremove_wake_function+0x0/0x40 2013-03-06 09:18:40 [] target_recovery_thread+0x566/0x1880 [ptlrpc] 2013-03-06 09:18:40 [] ? target_recovery_thread+0x0/0x1880 [ptlrpc] 2013-03-06 09:18:40 [] child_rip+0xa/0x20 2013-03-06 09:18:40 [] ? target_recovery_thread+0x0/0x1880 [ptlrpc] 2013-03-06 09:18:40 [] ? target_recovery_thread+0x0/0x1880 [ptlrpc] 2013-03-06 09:18:40 [] ? child_rip+0x0/0x20 2013-03-06 09:19:13 Lustre: ptmp2-MDT0000: recovery is timed out, evict stale exports 2013-03-06 09:19:13 Lustre: ptmp2-MDT0000: disconnecting 13 stale clients 2013-03-06 09:19:13 Lustre: ptmp2-MDT0000: sending delayed replies to recovered clients 2013-03-06 09:19:14 Lustre: MDS mdd_obd-ptmp2-MDT0000: ptmp2-OST00cb_UUID now active, resetting orphans 2013-03-06 09:19:14 Lustre: Skipped 15 previous similar messages 2013-03-06 09:19:14 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:27 Lustre: MDS mdd_obd-ptmp2-MDT0000: ptmp2-OST003d_UUID now active, resetting orphans 2013-03-06 09:19:27 Lustre: Skipped 75 previous similar messages 2013-03-06 09:19:29 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:29 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:29 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 1 previous similar message 2013-03-06 09:19:31 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:33 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:33 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 4 previous similar messages 2013-03-06 09:19:33 Lustre: 22321:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 7223ceb2-d7dd-364c-3844-8cbad769d4ca@JO.BOB.WI.TP@o2ib11 t0 exp (null) cur 1362557973 last 0 2013-03-06 09:19:33 Lustre: 22321:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 2 previous similar messages 2013-03-06 09:19:37 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:37 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 9 previous similar messages 2013-03-06 09:19:45 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Found orphan! Delete it 2013-03-06 09:19:45 Lustre: 22320:0:(mdd_orphans.c:371:orph_key_test_and_del()) Skipped 48 previous similar messages 2013-03-06 09:55:01 Lustre: DEBUG MARKER: Wed Mar 6 09:55:01 2013 2013-03-06 09:55:01 Console [lascaux111] log at 2013-03-06 10:00:00 CET. 2013-03-06 10:00:01 Lustre: DEBUG MARKER: Wed Mar 6 10:00:01 2013 2013-03-06 10:00:01 2013-03-06 10:05:01 Lustre: DEBUG MARKER: Wed Mar 6 10:05:01 2013 2013-03-06 10:05:01 2013-03-06 10:10:01 Lustre: DEBUG MARKER: Wed Mar 6 10:10:01 2013 2013-03-06 10:10:01 2013-03-06 10:15:01 Lustre: DEBUG MARKER: Wed Mar 6 10:15:01 2013 2013-03-06 10:15:01 2013-03-06 10:20:01 Lustre: DEBUG MARKER: Wed Mar 6 10:20:01 2013 2013-03-06 10:20:01 2013-03-06 10:20:11 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 3b7699e5-0960-d3e9-f00f-47fafb53ddb6@JO.BOO.WL.BWB@o2ib10 t0 exp (null) cur 1362561611 last 0 2013-03-06 10:20:11 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1 previous similar message 2013-03-06 10:20:22 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 3c941ef1-e4fe-deba-8f0f-becae95a7bf8@JO.BOB.WL.BWB@o2ib11 t0 exp (null) cur 1362561622 last 0 2013-03-06 10:20:22 Lustre: 22322:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 8 previous similar messages 2013-03-06 10:21:39 Lustre: 23009:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from dad03897-70e4-895c-304f-942bd238b623@JO.BOO.WI.AB@o2ib10 t0 exp (null) cur 1362561699 last 0 2013-03-06 10:21:39 Lustre: 23009:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 7 previous similar messages 2013-03-06 10:23:30 Lustre: 8377:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from e0d79cbd-b608-96c4-d192-d20b5d0eaab1@JO.BOO.WI.BLB@o2ib10 t0 exp (null) cur 1362561810 last 0 2013-03-06 10:23:30 Lustre: 8377:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 32 previous similar messages 2013-03-06 10:25:01 Lustre: DEBUG MARKER: Wed Mar 6 10:25:01 2013 2013-03-06 10:25:01 2013-03-06 10:26:30 Lustre: 7783:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from f117fa7a-fffa-7802-6a2e-894c4a5ccaee@JO.BOO.WA.ZI@o2ib10 t0 exp (null) cur 1362561990 last 0 2013-03-06 10:26:30 Lustre: 7783:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 231 previous similar messages 2013-03-06 10:30:01 Lustre: DEBUG MARKER: Wed Mar 6 10:30:01 2013 2013-03-06 10:30:01 2013-03-06 10:31:35 LustreError: 21487:0:(o2iblnd_cb.c:2992:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds 2013-03-06 10:31:35 LustreError: 21487:0:(o2iblnd_cb.c:3055:kiblnd_check_conns()) Timed out RDMA with JO.BOB.PA.BZI@o2ib11 (154): c: 7, oc: 0, rc: 8 2013-03-06 10:32:48 Lustre: ptmp2-MDT0000: haven't heard from client 96a55b69-2a68-fcda-3ffa-fb19fce57692 (at JO.BOB.PA.BZI@o2ib11) in 227 seconds. I think it's dead, and I am evicting it. exp ffff880c5a7a7400, cur 1362562368 expire 1362562218 last 1362562141 2013-03-06 10:32:48 Lustre: spool2-MDT0000: haven't heard from client 6f67f530-0caa-a15a-843c-b195d51f97ad (at JO.BOO.PA.BZI@o2ib10) in 227 seconds. I think it's dead, and I am evicting it. exp ffff880442907800, cur 1362562368 expire 1362562218 last 1362562141 2013-03-06 10:35:01 Lustre: DEBUG MARKER: Wed Mar 6 10:35:01 2013 2013-03-06 10:35:01 2013-03-06 10:40:01 Lustre: DEBUG MARKER: Wed Mar 6 10:40:01 2013 2013-03-06 10:40:01 2013-03-06 10:43:57 Lustre: 23009:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 13b81392-0e02-ace7-43a0-965c55267762@JO.BOO.PA.B@o2ib10 t0 exp (null) cur 1362563037 last 0 2013-03-06 10:43:57 Lustre: 23009:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 595 previous similar messages 2013-03-06 10:45:01 Lustre: DEBUG MARKER: Wed Mar 6 10:45:01 2013 2013-03-06 10:45:01 2013-03-06 10:45:34 Lustre: 8376:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 98678999-14d0-a0cf-523d-f61bd9b5a8f2@JO.BOO.PA.BBP@o2ib10 t0 exp (null) cur 1362563134 last 0 2013-03-06 10:45:34 Lustre: 8376:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 4 previous similar messages 2013-03-06 10:50:01 Lustre: DEBUG MARKER: Wed Mar 6 10:50:01 2013 2013-03-06 10:50:01 2013-03-06 10:55:01 Lustre: DEBUG MARKER: Wed Mar 6 10:55:01 2013 2013-03-06 10:55:01 Console [lascaux111] log at 2013-03-06 11:00:00 CET. 2013-03-06 11:00:01 Lustre: DEBUG MARKER: Wed Mar 6 11:00:01 2013 2013-03-06 11:00:01 2013-03-06 11:03:47 Lustre: spool2-MDT0000: haven't heard from client 8986f5be-3b3a-e95e-60f6-43de5b4f795d (at JO.BOO.PA.BFB@o2ib10) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8804428f8800, cur 1362564227 expire 1362564077 last 1362564000 2013-03-06 11:03:48 Lustre: ptmp2-MDT0000: haven't heard from client 4dd1b2ad-41b5-9bec-6b70-513f050041d6 (at JO.BOB.PA.BFB@o2ib11) in 228 seconds. I think it's dead, and I am evicting it. exp ffff880c5a7a1400, cur 1362564228 expire 1362564078 last 1362564000 2013-03-06 11:05:01 Lustre: DEBUG MARKER: Wed Mar 6 11:05:01 2013 2013-03-06 11:05:01 2013-03-06 11:10:01 Lustre: DEBUG MARKER: Wed Mar 6 11:10:01 2013 2013-03-06 11:10:01 2013-03-06 11:15:02 Lustre: DEBUG MARKER: Wed Mar 6 11:15:01 2013 2013-03-06 11:15:02 2013-03-06 11:20:01 Lustre: DEBUG MARKER: Wed Mar 6 11:20:01 2013 2013-03-06 11:20:01 2013-03-06 11:24:24 LustreError: 21487:0:(o2iblnd_cb.c:2992:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds 2013-03-06 11:24:24 LustreError: 21487:0:(o2iblnd_cb.c:3055:kiblnd_check_conns()) Timed out RDMA with JO.BOO.PA.BFB@o2ib10 (151): c: 7, oc: 0, rc: 8 2013-03-06 11:25:01 Lustre: DEBUG MARKER: Wed Mar 6 11:25:01 2013 2013-03-06 11:25:01 2013-03-06 11:30:01 Lustre: DEBUG MARKER: Wed Mar 6 11:30:01 2013 2013-03-06 11:30:01 2013-03-06 11:35:01 Lustre: DEBUG MARKER: Wed Mar 6 11:35:01 2013 2013-03-06 11:35:01 2013-03-06 11:40:01 Lustre: DEBUG MARKER: Wed Mar 6 11:40:01 2013 2013-03-06 11:40:01 2013-03-06 11:45:01 Lustre: DEBUG MARKER: Wed Mar 6 11:45:01 2013 2013-03-06 11:45:01 2013-03-06 11:50:01 Lustre: DEBUG MARKER: Wed Mar 6 11:50:01 2013 2013-03-06 11:50:01 2013-03-06 11:55:01 Lustre: DEBUG MARKER: Wed Mar 6 11:55:01 2013 2013-03-06 11:55:01 Console [lascaux111] log at 2013-03-06 12:00:00 CET. 2013-03-06 12:00:01 Lustre: DEBUG MARKER: Wed Mar 6 12:00:01 2013 2013-03-06 12:00:01 2013-03-06 12:05:02 Lustre: DEBUG MARKER: Wed Mar 6 12:05:01 2013 2013-03-06 12:05:02 2013-03-06 12:07:12 Lustre: 9773:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 0eed9e07-de8d-5e93-2679-e9cf16acaf75@JO.BOO.WZ.LWO@o2ib10 t0 exp (null) cur 1362568032 last 0 2013-03-06 12:07:12 Lustre: 9773:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 339 previous similar messages 2013-03-06 12:07:34 Lustre: 8377:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 51c79eb5-3b45-0948-328b-203420405cfd@JO.BOB.WZ.L@o2ib11 t0 exp (null) cur 1362568054 last 0 2013-03-06 12:07:34 Lustre: 8377:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 288 previous similar messages 2013-03-06 12:10:02 Lustre: DEBUG MARKER: Wed Mar 6 12:10:01 2013 2013-03-06 12:10:02 2013-03-06 12:15:01 Lustre: DEBUG MARKER: Wed Mar 6 12:15:01 2013 2013-03-06 12:15:01 2013-03-06 12:20:01 Lustre: DEBUG MARKER: Wed Mar 6 12:20:01 2013 2013-03-06 12:20:01 2013-03-06 12:25:01 Lustre: DEBUG MARKER: Wed Mar 6 12:25:01 2013 2013-03-06 12:25:01 2013-03-06 12:26:05 LustreError: 0:0:(ldlm_lockd.c:358:waiting_locks_callback()) ### lock callback timer expired after 2527s: evicting client at JO.BOB.WL.BWL@o2ib11 ns: mdt-ffff880c75791000 lock: ffff880f808fa240/0x10d7a36b2af44075 lrc: 3/0,0 mode: PR/PR res: 8941947168/295 bits 0x3 rrc: 2 type: IBT flags: 0x4000020 remote: 0x509d3a4be853f2eb expref: 204220 pid: 8377 timeout: 4301606895 2013-03-06 12:26:06 LustreError: 8459:0:(mdt_handler.c:2819:mdt_recovery()) operation 501 on unconnected MDS from 12345-JO.BOB.WL.BWL@o2ib11 2013-03-06 12:26:06 Lustre: 8459:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 90593929-8be2-3664-d338-40ad3d5e961d@JO.BOB.WL.BWL@o2ib11 t128849019945 exp (null) cur 1362569166 last 0 2013-03-06 12:26:06 Lustre: 8459:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 287 previous similar messages 2013-03-06 12:30:01 Lustre: DEBUG MARKER: Wed Mar 6 12:30:01 2013 2013-03-06 12:30:01 2013-03-06 12:35:01 Lustre: DEBUG MARKER: Wed Mar 6 12:35:01 2013 2013-03-06 12:35:01 2013-03-06 12:40:01 Lustre: DEBUG MARKER: Wed Mar 6 12:40:01 2013 2013-03-06 12:40:01 2013-03-06 12:45:01 Lustre: DEBUG MARKER: Wed Mar 6 12:45:01 2013 2013-03-06 12:45:01 2013-03-06 12:50:01 Lustre: DEBUG MARKER: Wed Mar 6 12:50:01 2013 2013-03-06 12:50:01 2013-03-06 12:55:01 Lustre: DEBUG MARKER: Wed Mar 6 12:55:01 2013 2013-03-06 12:55:01 Console [lascaux111] log at 2013-03-06 13:00:00 CET. 2013-03-06 13:00:01 Lustre: DEBUG MARKER: Wed Mar 6 13:00:01 2013 2013-03-06 13:00:01 2013-03-06 13:05:01 Lustre: DEBUG MARKER: Wed Mar 6 13:05:01 2013 2013-03-06 13:05:01 2013-03-06 13:10:01 Lustre: DEBUG MARKER: Wed Mar 6 13:10:01 2013 2013-03-06 13:10:01 2013-03-06 13:15:01 Lustre: DEBUG MARKER: Wed Mar 6 13:15:01 2013 2013-03-06 13:15:01 2013-03-06 13:20:01 Lustre: DEBUG MARKER: Wed Mar 6 13:20:01 2013 2013-03-06 13:20:01 2013-03-06 13:25:01 Lustre: DEBUG MARKER: Wed Mar 6 13:25:01 2013 2013-03-06 13:25:01 2013-03-06 13:30:01 Lustre: DEBUG MARKER: Wed Mar 6 13:30:01 2013 2013-03-06 13:30:01 2013-03-06 13:35:02 Lustre: DEBUG MARKER: Wed Mar 6 13:35:01 2013 2013-03-06 13:35:02 2013-03-06 13:37:41 Lustre: 9775:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from e59ecedb-ebc2-b888-ba11-2b20b7df5a87@JO.BOO.IO.LLF@o2ib10 t0 exp (null) cur 1362573461 last 0 2013-03-06 13:37:41 Lustre: 8634:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 44468a25-caba-7baa-6c2d-33abe86a3b26@JO.BOO.IO.BIP@o2ib10 t0 exp (null) cur 1362573461 last 0 2013-03-06 13:37:41 Lustre: 8634:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 25 previous similar messages 2013-03-06 13:37:42 Lustre: 23009:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from d2a81995-a596-49a1-5a4d-2b5eb8dbd5a9@JO.BOO.IB.IA@o2ib10 t0 exp (null) cur 1362573462 last 0 2013-03-06 13:37:42 Lustre: 23009:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 117 previous similar messages 2013-03-06 13:37:43 Lustre: 28966:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 72d88163-46de-1e67-886a-304cf898b97c@JO.BOO.IO.P@o2ib10 t0 exp (null) cur 1362573463 last 0 2013-03-06 13:37:43 Lustre: 28966:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 108 previous similar messages 2013-03-06 13:38:03 Lustre: 8375:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from e8686b1e-3362-abc8-2614-8a2f1fd1fab7@JO.BOB.IO.LBA@o2ib11 t0 exp (null) cur 1362573483 last 0 2013-03-06 13:38:03 Lustre: 8375:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 47 previous similar messages 2013-03-06 13:40:02 Lustre: DEBUG MARKER: Wed Mar 6 13:40:01 2013 2013-03-06 13:40:02 2013-03-06 13:41:19 Lustre: 28968:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 7e234f94-4ff6-69e6-d193-24cbfbdc8424@JO.BOO.IL.BWT@o2ib10 t0 exp (null) cur 1362573679 last 0 2013-03-06 13:41:19 Lustre: 28968:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 296 previous similar messages 2013-03-06 13:41:40 Lustre: 8801:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 2970b2f7-2b5d-246a-d3d5-22e13bed6ba6@JO.BOB.IL.FT@o2ib11 t0 exp (null) cur 1362573700 last 0 2013-03-06 13:41:40 Lustre: 8801:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 293 previous similar messages 2013-03-06 13:44:57 Lustre: 28966:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from a5799109-599a-a274-5977-2ab4c0849a15@JO.BOO.II.BPA@o2ib10 t0 exp (null) cur 1362573897 last 0 2013-03-06 13:44:57 Lustre: 28966:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 295 previous similar messages 2013-03-06 13:45:01 Lustre: DEBUG MARKER: Wed Mar 6 13:45:01 2013 2013-03-06 13:45:01 2013-03-06 13:48:36 Lustre: 8365:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 28bee1de-bcc2-0c52-0f01-4aa733fc1448@JO.BOO.IA.BFL@o2ib10 t0 exp (null) cur 1362574116 last 0 2013-03-06 13:48:36 Lustre: 8365:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 575 previous similar messages 2013-03-06 13:50:01 Lustre: DEBUG MARKER: Wed Mar 6 13:50:01 2013 2013-03-06 13:50:01 2013-03-06 13:53:11 Lustre: 7627:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 2c42bee9-0931-23a5-148a-0d4ee6c27231@JO.BOO.IZ.BOO@o2ib10 t0 exp (null) cur 1362574391 last 0 2013-03-06 13:53:11 Lustre: 7627:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 599 previous similar messages 2013-03-06 13:55:01 Lustre: DEBUG MARKER: Wed Mar 6 13:55:01 2013 2013-03-06 13:55:01 2013-03-06 13:56:48 Lustre: 8800:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from b3595709-9ab0-53f0-4535-396347a2058f@JO.BOO.PO.LBW@o2ib10 t0 exp (null) cur 1362574608 last 0 2013-03-06 13:56:48 Lustre: 8800:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 584 previous similar messages Console [lascaux111] log at 2013-03-06 14:00:00 CET. 2013-03-06 14:00:01 Lustre: DEBUG MARKER: Wed Mar 6 14:00:01 2013 2013-03-06 14:00:01 2013-03-06 14:04:20 Lustre: 28969:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 63ebd1cd-4315-42be-0b3e-d3b424446c81@JO.BOO.PI.BTZ@o2ib10 t0 exp (null) cur 1362575060 last 0 2013-03-06 14:04:20 Lustre: 28969:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1088 previous similar messages 2013-03-06 14:05:01 Lustre: DEBUG MARKER: Wed Mar 6 14:05:01 2013 2013-03-06 14:05:01 2013-03-06 14:10:01 Lustre: DEBUG MARKER: Wed Mar 6 14:10:01 2013 2013-03-06 14:10:01 2013-03-06 14:15:01 Lustre: DEBUG MARKER: Wed Mar 6 14:15:01 2013 2013-03-06 14:15:01 2013-03-06 14:15:14 Lustre: 8377:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 27720b5a-8d95-23b4-2b37-e282970852e4@JO.BOO.IO.PZ@o2ib10 t0 exp (null) cur 1362575714 last 0 2013-03-06 14:15:14 Lustre: 8377:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 612 previous similar messages 2013-03-06 14:20:01 Lustre: DEBUG MARKER: Wed Mar 6 14:20:01 2013 2013-03-06 14:20:01 2013-03-06 14:25:01 Lustre: DEBUG MARKER: Wed Mar 6 14:25:01 2013 2013-03-06 14:25:01 2013-03-06 14:26:18 Lustre: 7626:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from f4283e9d-bd08-6202-3474-1435291e77a0@JO.BOO.II.LT@o2ib10 t0 exp (null) cur 1362576378 last 0 2013-03-06 14:26:18 Lustre: 7626:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 6 previous similar messages 2013-03-06 14:30:02 Lustre: DEBUG MARKER: Wed Mar 6 14:30:01 2013 2013-03-06 14:30:02 2013-03-06 14:35:01 Lustre: DEBUG MARKER: Wed Mar 6 14:35:01 2013 2013-03-06 14:35:01 2013-03-06 14:40:01 Lustre: DEBUG MARKER: Wed Mar 6 14:40:01 2013 2013-03-06 14:40:01 2013-03-06 14:45:01 Lustre: DEBUG MARKER: Wed Mar 6 14:45:01 2013 2013-03-06 14:45:01 2013-03-06 14:50:01 Lustre: DEBUG MARKER: Wed Mar 6 14:50:01 2013 2013-03-06 14:50:01 2013-03-06 14:55:01 Lustre: DEBUG MARKER: Wed Mar 6 14:55:01 2013 2013-03-06 14:55:01 Console [lascaux111] log at 2013-03-06 15:00:00 CET. 2013-03-06 15:00:01 Lustre: DEBUG MARKER: Wed Mar 6 15:00:01 2013 2013-03-06 15:00:01 2013-03-06 15:05:01 Lustre: DEBUG MARKER: Wed Mar 6 15:05:01 2013 2013-03-06 15:05:01 2013-03-06 15:10:01 Lustre: DEBUG MARKER: Wed Mar 6 15:10:01 2013 2013-03-06 15:10:01 2013-03-06 15:15:01 Lustre: DEBUG MARKER: Wed Mar 6 15:15:01 2013 2013-03-06 15:15:01 2013-03-06 15:20:01 Lustre: DEBUG MARKER: Wed Mar 6 15:20:01 2013 2013-03-06 15:20:01 2013-03-06 15:25:01 Lustre: DEBUG MARKER: Wed Mar 6 15:25:01 2013 2013-03-06 15:25:01 2013-03-06 15:30:00 Lustre: DEBUG MARKER: Wed Mar 6 15:30:01 2013 2013-03-06 15:30:00 2013-03-06 15:33:46 Lustre: 14699:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from a1dcd0a1-c110-d99e-ec75-6847c9318109@JO.BOO.WI.PA@o2ib10 t0 exp (null) cur 1362580426 last 0 2013-03-06 15:33:46 Lustre: 14699:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 9 previous similar messages 2013-03-06 15:35:01 Lustre: DEBUG MARKER: Wed Mar 6 15:35:01 2013 2013-03-06 15:35:01 2013-03-06 15:40:01 Lustre: DEBUG MARKER: Wed Mar 6 15:40:01 2013 2013-03-06 15:40:01 2013-03-06 15:40:11 Lustre: 14498:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 1011374f-feee-8230-bf85-69b5129b5c58@JO.BOO.IZ.FF@o2ib10 t0 exp (null) cur 1362580811 last 0 2013-03-06 15:40:11 Lustre: 14498:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 9 previous similar messages 2013-03-06 15:45:01 Lustre: DEBUG MARKER: Wed Mar 6 15:45:01 2013 2013-03-06 15:45:01 2013-03-06 15:50:01 Lustre: DEBUG MARKER: Wed Mar 6 15:50:01 2013 2013-03-06 15:50:01 2013-03-06 15:55:01 Lustre: DEBUG MARKER: Wed Mar 6 15:55:01 2013 2013-03-06 15:55:01 Console [lascaux111] log at 2013-03-06 16:00:00 CET. 2013-03-06 16:00:01 Lustre: DEBUG MARKER: Wed Mar 6 16:00:01 2013 2013-03-06 16:00:01 2013-03-06 16:00:12 LustreError: 14650:0:(mdt_open.c:1023:mdt_reconstruct_open()) ASSERTION( (!(rc < 0) || (lustre_msg_get_transno(req->rq_repmsg) == 0)) ) failed: 2013-03-06 16:00:12 LustreError: 14650:0:(mdt_open.c:1023:mdt_reconstruct_open()) LBUG 2013-03-06 16:00:12 Pid: 14650, comm: mdt_119 2013-03-06 16:00:12 2013-03-06 16:00:12 Call Trace: 2013-03-06 16:00:12 [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] 2013-03-06 16:00:12 [] lbug_with_loc+0x47/0xb0 [libcfs] 2013-03-06 16:00:12 [] mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 16:00:12 [] mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 16:00:12 [] mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 16:00:12 [] mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 16:00:12 [] mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 16:00:12 [] ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 16:00:12 [] ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 16:00:12 [] mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 16:00:12 [] mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 16:00:12 [] mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 16:00:12 [] ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? __switch_to+0x1ac/0x320 2013-03-06 16:00:12 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] child_rip+0xa/0x20 2013-03-06 16:00:12 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? child_rip+0x0/0x20 2013-03-06 16:00:12 2013-03-06 16:00:12 Kernel panic - not syncing: LBUG 2013-03-06 16:00:12 Pid: 14650, comm: mdt_119 Not tainted 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 #1 2013-03-06 16:00:12 Call Trace: 2013-03-06 16:00:12 [] ? panic+0x78/0x143 2013-03-06 16:00:12 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] 2013-03-06 16:00:12 [] ? mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 16:00:12 [] ? mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 16:00:12 [] ? mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 16:00:12 [] ? mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 16:00:12 [] ? mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 16:00:12 [] ? ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 16:00:12 [] ? ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 16:00:12 [] ? mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 16:00:12 [] ? mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 16:00:12 [] ? mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 16:00:12 [] ? ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? __switch_to+0x1ac/0x320 2013-03-06 16:00:12 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? child_rip+0xa/0x20 2013-03-06 16:00:12 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 16:00:12 [] ? child_rip+0x0/0x20 2013-03-06 16:00:12 Initializing cgroup subsys cpuset 2013-03-06 16:00:12 Initializing cgroup subsys cpu 2013-03-06 16:00:12 Linux version 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 (efix@atlas.frec.bull.fr) (gcc version 4.4.5 20110214 (Bull 4.4.5-6) (GCC) ) #1 SMP Thu Jul 5 16:46:35 CEST 2012 2013-03-06 16:00:12 Command line: ro root=UUID=0d48d0fd-0d37-4272-848c-db985350b313 rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=C SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us console=tty0 console=ttyS1,115200 rdloaddriver=ahci rdloaddriver=aacraid rdloaddriver=megaraid_sas selinux=0 rdblacklist=lpfc transparent_hugepage=never pciehp_disable nmi_watchdog=0 nointremap intel_iommu=off audit=0 mce=0 irqpoll maxcpus=1 reset_devices nointremap intel_iommu=off memmap=exactmap memmap=624K@4K memmap=195964K@33392K elfcorehdr=229356K memmap=4K$0K memmap=12K$628K memmap=128K$896K memmap=10240K#1520344K memmap=512K$1564156K memmap=5740K#1564668K memmap=1172K#1570408K memmap=256K#1571580K memmap=1024K#1571836K memmap=278528K$2080768K memmap=16384K$4128768K memmap=16K$4174960K memmap=8192K$4186112K :::::::::::::: mdt_reconstruct_open/mdt_reconstruct_open.txt.4 :::::::::::::: 2013-03-06 16:58:10 Lustre: Lustre: Build Version: T-2_1_4_0-lustrebull-EFIX_319613_AE2_2-20130227105145-CHANGED-2.6.32-220.23.1.bl6.Bull.28.8.x86_64 2013-03-06 16:58:11 LustreError: 16255:0:(o2iblnd_cb.c:2248:kiblnd_passive_connect()) Can't accept JO.BOO.PL.BZZ@o2ib10 on NA (ib0:0:JO.BOO.WL.IL): bad dst nid JO.BOO.WL.IL@o2ib10 2013-03-06 16:58:11 LustreError: 16239:0:(o2iblnd_cb.c:2248:kiblnd_passive_connect()) Can't accept JO.BOO.IO.BL@o2ib10 on NA (ib0:0:JO.BOO.WL.IL): bad dst nid JO.BOO.WL.IL@o2ib10 2013-03-06 16:58:11 Lustre: Added LNI JO.BOO.WL.IL@o2ib10 [8/64/0/180] 2013-03-06 16:58:11 LustreError: 16239:0:(o2iblnd_cb.c:2248:kiblnd_passive_connect()) Skipped 1 previous similar message 2013-03-06 16:58:11 Lustre: Added LNI JO.BOB.WL.IL@o2ib11 [8/64/0/180] 2013-03-06 16:58:12 Lustre: Lustre OSC module (ffffffffa0a62100). 2013-03-06 16:58:12 Lustre: Lustre LOV module (ffffffffa0af5ce0). 2013-03-06 16:58:12 Lustre: Lustre client module (ffffffffa0be0960). 2013-03-06 16:58:13 LDISKFS-fs warning (device dm-0): ldiskfs_fill_super: 2013-03-06 16:58:13 LDISKFS-fs warning (device dm-2): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:13 extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:13 LDISKFS-fs (dm-2): barriers disabled 2013-03-06 16:58:13 LDISKFS-fs (dm-0): barriers disabled 2013-03-06 16:58:13 LDISKFS-fs (dm-2): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:13 LDISKFS-fs warning (device dm-2): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:13 LDISKFS-fs (dm-2): barriers disabled 2013-03-06 16:58:14 LDISKFS-fs (dm-0): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:14 LDISKFS-fs warning (device dm-0): ldiskfs_fill_super: extents feature not enabled on this filesystem, use tune2fs. 2013-03-06 16:58:14 LDISKFS-fs (dm-0): barriers disabled 2013-03-06 16:58:14 LDISKFS-fs (dm-2): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:14 LDISKFS-fs (dm-0): mounted filesystem with ordered data mode. Opts: 2013-03-06 16:58:14 LustreError: 27132:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff8807d02a9c00 x1428774442762243/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IB@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1 2013-03-06 16:58:14 LustreError: 27085:0:(ldlm_resource.c:751:ldlm_resource_complain()) Namespace MGCJO.BOO.WL.IB@o2ib10 resource refcount nonzero (1) after lock cleanup; forcing cleanup. 2013-03-06 16:58:14 LustreError: 27085:0:(ldlm_resource.c:757:ldlm_resource_complain()) Resource: ffff8807b7829e40 (216634586224/0/0/0) (rc: 1) 2013-03-06 16:58:14 LustreError: 27132:0:(client.c:1076:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff8807d02a9c00 x1428774442762244/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@JO.BOO.WL.IB@o2ib10:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:/0/ffffffff rc 0/-1 2013-03-06 16:58:14 Lustre: MGCJO.BOO.WL.IB@o2ib10: Reactivating import 2013-03-06 16:58:15 Lustre: Enabling ACL 2013-03-06 16:58:15 Lustre: Enabling user_xattr 2013-03-06 16:58:17 Lustre: mdd_obd-ptmp2-MDT0000: changelog on 2013-03-06 16:58:17 Lustre: ptmp2-MDT0000: used disk, loading 2013-03-06 16:58:17 Lustre: 27974:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service ptmp2-MDT0000, 3394 recoverable clients, last_transno 133143987287 2013-03-06 16:58:17 LustreError: 137-5: ptmp2-MDT0000: Not available for connect from JO.BOB.WF.AF@o2ib11 (not set up) 2013-03-06 16:58:17 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:17 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:17 LustreError: Skipped 1 previous similar message 2013-03-06 16:58:17 LustreError: 137-5: ptmp2-MDT0000: Not available for connect from JO.BOB.PP.BI@o2ib11 (not set up) 2013-03-06 16:58:18 LustreError: Skipped 1 previous similar message 2013-03-06 16:58:18 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.PO.LWB@o2ib11 2013-03-06 16:58:18 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.PO.ZZ@o2ib11 2013-03-06 16:58:18 Lustre: Skipped 3 previous similar messages 2013-03-06 16:58:18 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:18 LustreError: Skipped 67 previous similar messages 2013-03-06 16:58:18 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.IO.BLA@o2ib11 2013-03-06 16:58:18 Lustre: Skipped 26 previous similar messages 2013-03-06 16:58:19 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:19 LustreError: Skipped 337 previous similar messages 2013-03-06 16:58:19 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.IA.BFA@o2ib11 2013-03-06 16:58:19 Lustre: Skipped 140 previous similar messages 2013-03-06 16:58:21 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:21 LustreError: Skipped 486 previous similar messages 2013-03-06 16:58:21 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.WA.BIZ@o2ib11 2013-03-06 16:58:21 Lustre: Skipped 279 previous similar messages 2013-03-06 16:58:26 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:26 LustreError: Skipped 776 previous similar messages 2013-03-06 16:58:26 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.IA.LZ@o2ib11 2013-03-06 16:58:26 Lustre: Skipped 333 previous similar messages 2013-03-06 16:58:26 LustreError: 27973:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880472425400 x1428774442762255/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:58:35 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:35 LustreError: Skipped 515 previous similar messages 2013-03-06 16:58:35 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.WT.AW@o2ib11 2013-03-06 16:58:35 Lustre: Skipped 316 previous similar messages 2013-03-06 16:58:51 LustreError: 137-5: UUID 'cprot2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:58:51 LustreError: Skipped 3397 previous similar messages 2013-03-06 16:58:51 Lustre: ptmp2-MDT0000: Temporarily refusing client connection from JO.BOB.PA.B@o2ib11 2013-03-06 16:58:51 Lustre: Skipped 2053 previous similar messages 2013-03-06 16:59:06 Lustre: 28285:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 1d939734-8681-2b85-6d8d-3159d0f0b8c7@JO.BOB.IB.WF@o2ib11 recovering/t128849578888 exp ffff8809f2253800 cur 1362585546 last 1362585497 2013-03-06 16:59:06 Lustre: 28285:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1 previous similar message 2013-03-06 16:59:06 Lustre: 28276:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 197d8ecb-a28c-4194-7390-00e1f31d2dee@JO.BOB.PI.LWB@o2ib11 recovering/t128853417579 exp ffff880b13d32000 cur 1362585546 last 1362585497 2013-03-06 16:59:06 Lustre: 28276:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 2 previous similar messages 2013-03-06 16:59:07 Lustre: 28271:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from d9fab0f0-c4d2-d024-51ca-88d6dd5322fd@JO.BOB.IO.BOB@o2ib11 recovering/t128852217366 exp ffff8809f2293c00 cur 1362585547 last 1362585497 2013-03-06 16:59:07 Lustre: 28271:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 66 previous similar messages 2013-03-06 16:59:07 Lustre: 27304:0:(client.c:1833:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1362585494/real 1362585494] req@ffff8810735e3800 x1428774442762245/t0(0) o250->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 368/512 e 0 to 1 dl 1362585499 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 2013-03-06 16:59:08 Lustre: 28278:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 9d86614e-424e-bbca-5001-a70af4dc1160@JO.BOB.IF.AZ@o2ib11 recovering/t128849631919 exp ffff8809f2927000 cur 1362585548 last 1362585497 2013-03-06 16:59:08 Lustre: 28278:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 100 previous similar messages 2013-03-06 16:59:10 Lustre: 28271:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from 43513a71-a904-d112-aa6f-d7c08556e301@JO.BOB.WP.TL@o2ib11 recovering/t128852945790 exp ffff880bb1e9b400 cur 1362585550 last 1362585497 2013-03-06 16:59:10 Lustre: 28271:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 446 previous similar messages 2013-03-06 16:59:12 LustreError: 27973:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff8804553db800 x1428774442762581/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:59:14 Lustre: 28143:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from ac6c8b6e-fe5e-4ad0-db0c-576efb6bf722@JO.BOB.WI.FB@o2ib11 recovering/t0 exp ffff880bb1e6ac00 cur 1362585554 last 1362585497 2013-03-06 16:59:14 Lustre: 28143:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 1067 previous similar messages 2013-03-06 16:59:17 LustreError: 27085:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff881072fe0800 x1428774442762582/t0(0) o253->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 4736/4736 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:59:23 Lustre: 28143:0:(ldlm_lib.c:952:target_handle_connect()) ptmp2-MDT0000: connection from d2b180be-a4d5-9a56-5342-001d6f4cce9a@JO.BOB.WZ.BWL@o2ib11 recovering/t128853861693 exp ffff880ac1962400 cur 1362585563 last 1362585497 2013-03-06 16:59:23 Lustre: 28143:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 483 previous similar messages 2013-03-06 16:59:23 LustreError: 137-5: UUID 'spool2-MDT0000_UUID' is not available for connect (no target) 2013-03-06 16:59:23 LustreError: Skipped 2094 previous similar messages 2013-03-06 16:59:23 LustreError: 28143:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff88052723d000 x1428756071456930/t0(128854126900) o101->f770fcca-c4c5-8d1b-a30d-4330bc04747d@JO.BOB.WZ.PB@o2ib11:0/0 lens 552/4936 e 0 to 0 dl 1362585604 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:59:23 LustreError: 28276:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff88052b973000 x1428756046290681/t0(128854126694) o101->e1d2c582-9e75-1a6c-8e54-f4337f7ca662@JO.BOB.WZ.PW@o2ib11:0/0 lens 552/4936 e 0 to 0 dl 1362585604 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:59:23 LustreError: 27085:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff88051fc84000 x1428774442762584/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:59:23 LustreError: 27085:0:(client.c:1065:ptlrpc_import_delay_req()) Skipped 1 previous similar message 2013-03-06 16:59:26 LustreError: 27973:0:(client.c:1065:ptlrpc_import_delay_req()) @@@ send limit expired req@ffff880232e23c00 x1428774442762585/t0(0) o101->MGCJO.BOO.WL.IB@o2ib10@0@lo:26/25 lens 296/352 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 2013-03-06 16:59:29 Lustre: Enabling ACL 2013-03-06 16:59:29 Lustre: Enabling user_xattr 2013-03-06 16:59:30 LustreError: 137-5: spool2-MDT0000: Not available for connect from JO.BOO.IO.LL@o2ib10 (not set up) 2013-03-06 16:59:30 LustreError: Skipped 1 previous similar message 2013-03-06 16:59:30 Lustre: spool2-MDT0000: used disk, loading 2013-03-06 16:59:30 Lustre: 28729:0:(ldlm_lib.c:2028:target_recovery_init()) RECOVERY: service spool2-MDT0000, 3396 recoverable clients, last_transno 120259205266 2013-03-06 16:59:30 Lustre: MGCJO.BOO.WL.IB@o2ib10: Reactivating import 2013-03-06 16:59:31 Lustre: spool2-MDT0000: Temporarily refusing client connection from JO.BOO.IO.BWW@o2ib10 2013-03-06 16:59:31 Lustre: Skipped 1030 previous similar messages 2013-03-06 16:59:39 Lustre: 28283:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 0a9ffb31-9279-11de-92f2-cb35da41ef4f@JO.BOO.II.BFZ@o2ib10 recovering/t0 exp ffff88096080d400 cur 1362585579 last 1362585570 2013-03-06 16:59:39 Lustre: 28283:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 2029 previous similar messages 2013-03-06 16:59:49 LustreError: 28735:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff88078cbf2c00 x1428761965157829/t0(120259189917) o101->9bb4f068-58bd-f4f4-be5d-b431acc1cb57@JO.BOO.IL.ZP@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362585630 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:59:51 LustreError: 29334:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff88080f0a3400 x1428763196028511/t0(120259166492) o101->3be280cc-98c7-32e5-459f-f930addf430e@JO.BOO.PW.PF@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362585632 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 16:59:51 LustreError: 29334:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 14 previous similar messages Console [lascaux111] log at 2013-03-06 17:00:00 CET. 2013-03-06 17:00:02 LustreError: 28140:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880846445400 x1428749746860601/t0(120259196573) o101->bd53603c-df05-e29b-b367-7944045d735c@JO.BOO.WA.WZ@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362585644 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:00:02 LustreError: 28140:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 29 previous similar messages 2013-03-06 17:00:08 LustreError: 28140:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff8807dc432000 x1428762205289484/t0(120259197459) o101->0c9a6153-aaba-0362-bf87-aa63103427d7@JO.BOO.II.LBZ@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362585649 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:00:11 Lustre: 28140:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from ddc6a157-30af-3e73-6547-91ebe45a202f@JO.BOO.PL.BOI@o2ib10 recovering/t120259202378 exp ffff88095bc5dc00 cur 1362585611 last 1362585570 2013-03-06 17:00:11 Lustre: 28140:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 2232 previous similar messages 2013-03-06 17:00:16 LustreError: 28291:0:(mdt_open.c:1281:mdt_reint_open()) @@@ OPEN & CREAT not in open replay. req@ffff880b985e4050 x1428763221198575/t0(120259171326) o101->118e6bc3-3cab-e6d6-3a11-8d0968d31cd9@JO.BOO.PW.AT@o2ib10:0/0 lens 536/1480 e 0 to 0 dl 1362585658 ref 1 fl Interpret:/4/0 rc 0/0 2013-03-06 17:00:16 LustreError: 28291:0:(mdt_open.c:1281:mdt_reint_open()) Skipped 28 previous similar messages 2013-03-06 17:02:36 Lustre: 28139:0:(ldlm_lib.c:952:target_handle_connect()) spool2-MDT0000: connection from 2de0a4b0-0882-9565-7793-bf1d9555c30f@JO.BOO.WL.BWL@o2ib10 recovering/t120259125941 exp ffff88096a2e8400 cur 1362585756 last 1362585570 2013-03-06 17:02:36 Lustre: 28139:0:(ldlm_lib.c:952:target_handle_connect()) Skipped 319 previous similar messages 2013-03-06 17:04:06 Lustre: ptmp2-MDT0000: recovery is timed out, evict stale exports 2013-03-06 17:04:06 Lustre: ptmp2-MDT0000: disconnecting 16 stale clients 2013-03-06 17:04:07 LustreError: 30591:0:(mdt_open.c:1023:mdt_reconstruct_open()) ASSERTION( (!(rc < 0) || (lustre_msg_get_transno(req->rq_repmsg) == 0)) ) failed: 2013-03-06 17:04:07 LustreError: 30591:0:(mdt_open.c:1023:mdt_reconstruct_open()) LBUG 2013-03-06 17:04:07 Pid: 30591, comm: mdt_164 2013-03-06 17:04:07 2013-03-06 17:04:07 Call Trace: 2013-03-06 17:04:07 [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] 2013-03-06 17:04:07 [] lbug_with_loc+0x47/0xb0 [libcfs] 2013-03-06 17:04:07 [] mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 17:04:07 [] mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 17:04:07 [] mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 17:04:07 [] mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 17:04:07 [] mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 17:04:07 [] ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 17:04:07 [] ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 17:04:07 [] mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 17:04:07 [] mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 17:04:07 [] mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 17:04:07 [] ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? __switch_to+0x1ac/0x320 2013-03-06 17:04:07 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] child_rip+0xa/0x20 2013-03-06 17:04:07 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? child_rip+0x0/0x20 2013-03-06 17:04:07 2013-03-06 17:04:07 Kernel panic - not syncing: LBUG 2013-03-06 17:04:07 Pid: 30591, comm: mdt_164 Not tainted 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 #1 2013-03-06 17:04:07 Call Trace: 2013-03-06 17:04:07 [] ? panic+0x78/0x143 2013-03-06 17:04:07 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] 2013-03-06 17:04:07 [] ? mdt_reconstruct_open+0x7c7/0xa80 [mdt] 2013-03-06 17:04:07 [] ? mdt_reconstruct+0x45/0x120 [mdt] 2013-03-06 17:04:07 [] ? mdt_reint_internal+0x709/0x8e0 [mdt] 2013-03-06 17:04:07 [] ? mdt_intent_reint+0x1ed/0x500 [mdt] 2013-03-06 17:04:07 [] ? mdt_intent_policy+0x379/0x690 [mdt] 2013-03-06 17:04:07 [] ? ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] 2013-03-06 17:04:07 [] ? ldlm_handle_enqueue0+0x48d/0xf50 [ptlrpc] 2013-03-06 17:04:07 [] ? mdt_enqueue+0x46/0x130 [mdt] 2013-03-06 17:04:07 [] ? mdt_handle_common+0x932/0x1750 [mdt] 2013-03-06 17:04:07 [] ? mdt_regular_handle+0x15/0x20 [mdt] 2013-03-06 17:04:07 [] ? ptlrpc_main+0xd16/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? __switch_to+0x1ac/0x320 2013-03-06 17:04:07 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? child_rip+0xa/0x20 2013-03-06 17:04:07 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? ptlrpc_main+0x0/0x1a80 [ptlrpc] 2013-03-06 17:04:07 [] ? child_rip+0x0/0x20 2013-03-06 17:04:07 Initializing cgroup subsys cpuset 2013-03-06 17:04:07 Initializing cgroup subsys cpu 2013-03-06 17:04:07 Linux version 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 (efix@atlas.frec.bull.fr) (gcc version 4.4.5 20110214 (Bull 4.4.5-6) (GCC) ) #1 SMP Thu Jul 5 16:46:35 CEST 2012 2013-03-06 17:04:07 Command line: ro root=UUID=0d48d0fd-0d37-4272-848c-db985350b313 rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=C SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us console=tty0 console=ttyS1,115200 rdloaddriver=ahci rdloaddriver=aacraid rdloaddriver=megaraid_sas selinux=0 rdblacklist=lpfc transparent_hugepage=never pciehp_disable nmi_watchdog=0 nointremap intel_iommu=off audit=0 mce=0 irqpoll maxcpus=1 reset_devices nointremap intel_iommu=off memmap=exactmap memmap=624K@4K memmap=195964K@33392K elfcorehdr=229356K memmap=4K$0K memmap=12K$628K memmap=128K$896K memmap=10240K#1520344K memmap=512K$1564156K memmap=5740K#1564668K memmap=1172K#1570408K memmap=256K#1571580K memmap=1024K#1571836K memmap=278528K$2080768K memmap=16384K$4128768K memmap=16K$4174960K memmap=8192K$4186112K :::::::::::::: trace_debug_lascaux110_mdt_reconstruct_open_assertion.txt :::::::::::::: [root@lascaux1064 127.0.0.1-2013-03-06-16:00:31] # EDITOR=emacs crash /usr/lib/debug/lib/modules/$(uname -r)/vmlinux vmcore crash 5.1.1-2.bl6 Copyright (C) 2002-2011 Red Hat, Inc. Copyright (C) 2004, 2005, 2006 IBM Corporation Copyright (C) 1999-2006 Hewlett-Packard Co Copyright (C) 2005, 2006 Fujitsu Limited Copyright (C) 2006, 2007 VA Linux Systems Japan K.K. Copyright (C) 2005 NEC Corporation Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc. Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc. This program is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Enter "help copying" to see the conditions. This program has absolutely no warranty. Enter "help warranty" for details. GNU gdb (GDB) 7.0 Copyright (C) 2009 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-unknown-linux-gnu"... KERNEL: /usr/lib/debug/lib/modules/2.6.32-220.23.1.bl6.Bull.28.8.x86_64/vmlinux DUMPFILE: vmcore [PARTIAL DUMP] CPUS: 32 DATE: Wed Mar 6 16:00:12 2013 UPTIME: 22:05:42 LOAD AVERAGE: 0.25, 0.12, 0.45 TASKS: 1860 NODENAME: lascaux111 RELEASE: 2.6.32-220.23.1.bl6.Bull.28.8.x86_64 VERSION: #1 SMP Thu Jul 5 16:46:35 CEST 2012 MACHINE: x86_64 (2267 Mhz) MEMORY: 64 GB PANIC: "Kernel panic - not syncing: LBUG" PID: 14650 COMMAND: "mdt_119" TASK: ffff88021e022100 [THREAD_INFO: ffff8802d4ce8000] CPU: 15 STATE: TASK_RUNNING (PANIC) crash> bt PID: 14650 TASK: ffff88021e022100 CPU: 15 COMMAND: "mdt_119" #0 [ffff8802d4ceb8f8] machine_kexec at ffffffff8102895b #1 [ffff8802d4ceb958] crash_kexec at ffffffff810a4622 #2 [ffff8802d4ceba28] panic at ffffffff81484657 #3 [ffff8802d4cebaa8] lbug_with_loc at ffffffffa0513e5b [libcfs] #4 [ffff8802d4cebac8] mdt_reconstruct_open at ffffffffa0d2ad87 [mdt] #5 [ffff8802d4cebb58] mdt_reconstruct at ffffffffa0d1c8c5 [mdt] #6 [ffff8802d4cebb78] mdt_reint_internal at ffffffffa0d09099 [mdt] #7 [ffff8802d4cebbc8] mdt_intent_reint at ffffffffa0d0953d [mdt] #8 [ffff8802d4cebc18] mdt_intent_policy at ffffffffa0d07c09 [mdt] #9 [ffff8802d4cebc58] ldlm_lock_enqueue at ffffffffa07203c1 [ptlrpc] #10 [ffff8802d4cebcb8] ldlm_handle_enqueue0 at ffffffffa07463dd [ptlrpc] #11 [ffff8802d4cebd28] mdt_enqueue at ffffffffa0d08586 [mdt] #12 [ffff8802d4cebd48] mdt_handle_common at ffffffffa0cfd762 [mdt] #13 [ffff8802d4cebd98] mdt_regular_handle at ffffffffa0cfe655 [mdt] #14 [ffff8802d4cebda8] ptlrpc_main at ffffffffa07754f6 [ptlrpc] #15 [ffff8802d4cebf48] kernel_thread at ffffffff8100412a crash> bt -f .......... #3 [ffff8802d4cebaa8] lbug_with_loc at ffffffffa0513e5b [libcfs] ffff8802d4cebab0: 000000001302baaa ffff880462ff0800 ffff8802d4cebac0: ffff8802d4cebb50 ffffffffa0d2ad87 #4 [ffff8802d4cebac8] mdt_reconstruct_open at ffffffffa0d2ad87 [mdt] ffff8802d4cebad0: ffff8802d4cebb60 ffff880462ff0800 ffff8802d4cebae0: ffff8804429d8000 ffff880d9abcab70 ffff8802d4cebaf0: ffff8802d4cebb10 0000000022000001 ffff8802d4cebb00: ffff880462ff08b0 ffff880f91890250 ffff8802d4cebb10: ffff880c75791000 ffff88060b574cc0 ffff8802d4cebb20: ffff8802d4cebb30 ffff880462ff0800 ffff8802d4cebb30: ffff880462ff08b0 0000000000000000 ffff8802d4cebb40: 0000000000000006 ffff880462ff08b0 ffff8802d4cebb50: ffff8802d4cebb70 ffffffffa0d1c8c5 #5 [ffff8802d4cebb58] mdt_reconstruct at ffffffffa0d1c8c5 [mdt] ffff8802d4cebb60: ffff880d9abca800 ffff880462ff0800 ffff8802d4cebb70: ffff8802d4cebbc0 ffffffffa0d09099 .......... void mdt_reconstruct_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc) { const struct lu_env *env = info->mti_env; struct mdt_device *mdt = info->mti_mdt; struct req_capsule *pill = info->mti_pill; struct ptlrpc_request *req = mdt_info_req(info); struct tg_export_data *ted = &req->rq_export->exp_target_data; struct lsd_client_data *lcd = ted->ted_lcd; struct md_attr *ma = &info->mti_attr; struct mdt_reint_record *rr = &info->mti_rr; __u32 flags = info->mti_spec.sp_cr_flags; struct ldlm_reply *ldlm_rep; struct mdt_object *parent; struct mdt_object *child; struct mdt_body *repbody; int rc; ENTRY; LASSERT(pill->rc_fmt == &RQF_LDLM_INTENT_OPEN); ldlm_rep = req_capsule_server_get(pill, &RMF_DLM_REP); repbody = req_capsule_server_get(pill, &RMF_MDT_BODY); ma->ma_lmm = req_capsule_server_get(pill, &RMF_MDT_MD); ma->ma_lmm_size = req_capsule_get_size(pill, &RMF_MDT_MD, RCL_SERVER); ma->ma_need = MA_INODE; if (ma->ma_lmm_size > 0) ma->ma_need |= MA_LOV; ma->ma_valid = 0; mdt_req_from_lcd(req, lcd); mdt_set_disposition(info, ldlm_rep, lcd->lcd_last_data); CDEBUG(D_INODE, "This is reconstruct open: disp="LPX64", result=%d\n", ldlm_rep->lock_policy_res1, req->rq_status); if (mdt_get_disposition(ldlm_rep, DISP_OPEN_CREATE) && req->rq_status != 0) /* We did not create successfully, return error to client. */ GOTO(out, rc = req->rq_status); if (mdt_get_disposition(ldlm_rep, DISP_OPEN_CREATE)) { struct obd_export *exp = req->rq_export; /* * We failed after creation, but we do not know in which step * we failed. So try to check the child object. */ parent = mdt_object_find(env, mdt, rr->rr_fid1); if (IS_ERR(parent)) { rc = PTR_ERR(parent); LCONSOLE_WARN("Parent "DFID" lookup error %d." " Evicting client %s with export %s.\n", PFID(rr->rr_fid1), rc, obd_uuid2str(&exp->exp_client_uuid), obd_export_nid2str(exp)); mdt_export_evict(exp); RETURN_EXIT; } child = mdt_object_find(env, mdt, rr->rr_fid2); if (IS_ERR(child)) { rc = PTR_ERR(child); LCONSOLE_WARN("Child "DFID" lookup error %d." " Evicting client %s with export %s.\n", PFID(mdt_object_fid(child)), rc, obd_uuid2str(&exp->exp_client_uuid), obd_export_nid2str(exp)); mdt_object_put(env, parent); mdt_export_evict(exp); RETURN_EXIT; } rc = mdt_object_exists(child); if (rc > 0) { struct md_object *next; mdt_set_capainfo(info, 1, rr->rr_fid2, BYPASS_CAPA); next = mdt_object_child(child); rc = mo_attr_get(env, next, ma); if (rc == 0) rc = mdt_finish_open(info, parent, child, flags, 1, ldlm_rep); } else if (rc < 0) { /* the child object was created on remote server */ repbody->fid1 = *rr->rr_fid2; repbody->valid |= (OBD_MD_FLID | OBD_MD_MDS); rc = 0; } else if (rc == 0) { /* the child does not exist, we should do regular open */ mdt_object_put(env, parent); mdt_object_put(env, child); GOTO(regular_open, 0); } mdt_object_put(env, parent); mdt_object_put(env, child); GOTO(out, rc); } else { regular_open: /* We did not try to create, so we are a pure open */ rc = mdt_reint_open(info, lhc); } EXIT; out: req->rq_status = rc; lustre_msg_set_status(req->rq_repmsg, req->rq_status); LASSERT(ergo(rc < 0, lustre_msg_get_transno(req->rq_repmsg) == 0)); } crash> disas mdt_reconstruct_open Dump of assembler code for function mdt_reconstruct_open: 0xffffffffa0d2a5c0 : push %rbp 0xffffffffa0d2a5c1 : mov %rsp,%rbp 0xffffffffa0d2a5c4 : push %r15 0xffffffffa0d2a5c6 : push %r14 0xffffffffa0d2a5c8 : push %r13 0xffffffffa0d2a5ca : push %r12 <---- ffff880462ff08b0 0xffffffffa0d2a5cc : push %rbx <---- ffff880462ff0800 0xffffffffa0d2a5cd : sub $0x58,%rsp 0xffffffffa0d2a5d1 : nopl 0x0(%rax,%rax,1) 0xffffffffa0d2a5d6 : xor %r12d,%r12d 0xffffffffa0d2a5d9 : mov %rsi,-0x50(%rbp) crash> disas mdt_reconstruct Dump of assembler code for function mdt_reconstruct: 0xffffffffa0d1c880 : push %rbp 0xffffffffa0d1c881 : mov %rsp,%rbp 0xffffffffa0d1c884 : sub $0x10,%rsp 0xffffffffa0d1c888 : mov %rbx,(%rsp) 0xffffffffa0d1c88c : mov %r12,0x8(%rsp) 0xffffffffa0d1c891 : nopl 0x0(%rax,%rax,1) 0xffffffffa0d1c896 : testb $0x1,-0x7e73b9(%rip) # 0xffffffffa05354e4 0xffffffffa0d1c89d : mov %rdi,%rbx 0xffffffffa0d1c8a0 : mov %rsi,%r12 0xffffffffa0d1c8a3 : je 0xffffffffa0d1c8b2 0xffffffffa0d1c8a5 : testb $0x4,-0x7e73cc(%rip) # 0xffffffffa05354e0 0xffffffffa0d1c8ac : jne 0xffffffffa0d1c940 0xffffffffa0d1c8b2 : mov 0x1f0(%rbx),%eax 0xffffffffa0d1c8b8 : mov %r12,%rsi <---- r12 struct mdt_lock_handle *lhc 0xffffffffa0d1c8bb : mov %rbx,%rdi <---- rbx struct mdt_thread_info *info 0xffffffffa0d1c8be : callq *-0x5f2cb7c0(,%rax,8) <----- ICI 0xffffffffa0d1c8c5 : testb $0x1,-0x7e73e8(%rip) # 0xffffffffa05354e4 0xffffffffa0d1c8cc : je 0xffffffffa0d1c92b 0xffffffffa0d1c8ce : testb $0x4,-0x7e73f5(%rip) # 0xffffffffa05354e0 0xffffffffa0d1c8d5 : je 0xffffffffa0d1c92b 0xffffffffa0d1c8d7 : mov $0xffffffffa0d3d0c0,%rsi 0xffffffffa0d1c8de : mov $0xffffffffa0d5b5c0,%rdi 0xffffffffa0d1c8e5 : xor %eax,%eax 0xffffffffa0d1c8e7 : movl $0x4,0x3ecdf(%rip) # 0xffffffffa0d5b5d0 0xffffffffa0d1c8f1 : movq $0xffffffffa0d38898,0x3ecc4(%rip) # 0xffffffffa0d5b5c0 0xffffffffa0d1c8fc : movq $0xffffffffa0d34880,0x3ecc1(%rip) # 0xffffffffa0d5b5c8 0xffffffffa0d1c907 : movl $0x4b0,0x3ecc3(%rip) # 0xffffffffa0d5b5d4 0xffffffffa0d1c911 : movq $0x0,0x3ecc4(%rip) # 0xffffffffa0d5b5e0 0xffffffffa0d1c91c : movl $0x1,0x3ecb2(%rip) # 0xffffffffa0d5b5d8 0xffffffffa0d1c926 : callq 0xffffffffa0520090 0xffffffffa0d1c92b : mov (%rsp),%rbx 0xffffffffa0d1c92f : mov 0x8(%rsp),%r12 0xffffffffa0d1c934 : leaveq 0xffffffffa0d1c935 : retq 0xffffffffa0d1c936 : nopw %cs:0x0(%rax,%rax,1) 0xffffffffa0d1c940 : mov $0xffffffffa0d3d0af,%rsi 0xffffffffa0d1c947 : mov $0xffffffffa0d5b600,%rdi 0xffffffffa0d1c94e : xor %eax,%eax 0xffffffffa0d1c950 : movl $0x4,0x3ecb6(%rip) # 0xffffffffa0d5b610 0xffffffffa0d1c95a : movq $0xffffffffa0d38898,0x3ec9b(%rip) # 0xffffffffa0d5b600 0xffffffffa0d1c965 : movq $0xffffffffa0d34880,0x3ec98(%rip) # 0xffffffffa0d5b608 0xffffffffa0d1c970 : movl $0x4ae,0x3ec9a(%rip) # 0xffffffffa0d5b614 0xffffffffa0d1c97a : movq $0x0,0x3ec9b(%rip) # 0xffffffffa0d5b620 0xffffffffa0d1c985 : movl $0x1,0x3ec89(%rip) # 0xffffffffa0d5b618 0xffffffffa0d1c98f : callq 0xffffffffa0520090 0xffffffffa0d1c994 : jmpq 0xffffffffa0d1c8b2 End of assembler dump. crash> struct mdt_thread_info ffff880462ff0800 struct mdt_thread_info { mti_pill = 0xffff880d9abcab70, mti_exp = 0xffff8806065c3800, mti_lh = {{ mlh_type = MDT_PDO_LOCK, mlh_reg_lh = { cookie = 0 }, mlh_reg_mode = LCK_PR, mlh_pdo_lh = { cookie = 0 }, mlh_pdo_mode = LCK_CR, mlh_pdo_hash = 601460990 }, { mlh_type = MDT_NUL_LOCK, mlh_reg_lh = { cookie = 0 }, mlh_reg_mode = LCK_MINMODE, mlh_pdo_lh = { cookie = 0 }, mlh_pdo_mode = LCK_MINMODE, mlh_pdo_hash = 0 }, { mlh_type = MDT_NUL_LOCK, mlh_reg_lh = { cookie = 0 }, mlh_reg_mode = LCK_MINMODE, mlh_pdo_lh = { cookie = 0 }, mlh_pdo_mode = LCK_MINMODE, mlh_pdo_hash = 0 }, { mlh_type = MDT_NUL_LOCK, mlh_reg_lh = { cookie = 0 }, mlh_reg_mode = LCK_MINMODE, mlh_pdo_lh = { cookie = 0 }, mlh_pdo_mode = LCK_MINMODE, mlh_pdo_hash = 0 }, { mlh_type = MDT_REG_LOCK, mlh_reg_lh = { cookie = 1213618305523011301 }, mlh_reg_mode = LCK_CR, mlh_pdo_lh = { cookie = 0 }, mlh_pdo_mode = LCK_MINMODE, mlh_pdo_hash = 0 }}, mti_mdt = 0xffff880c75791000, mti_env = 0xffff8802d4cebe90, mti_fail_id = 290, mti_transno = 0, mti_attr = { ma_valid = 3, ma_need = 3, ma_attr_flags = 0, ma_attr = { la_size = 0, la_mtime = 1362452637, la_atime = 1362582012, la_ctime = 1362452637, la_blocks = 8, la_mode = 33184, la_uid = 2078, la_gid = 3607, la_flags = 0, la_nlink = 1, la_blkbits = 12, la_blksize = 4096, la_rdev = 0, la_valid = 7935 }, ma_lmm = 0xffff880f91890328, ma_lmm_size = 3872, ma_lmv = 0x0, ma_lmv_size = 0, ma_acl = 0x0, ma_acl_size = 0, ma_cookie = 0x0, ma_cookie_size = 0, ma_capa = 0x0, ma_hsm = { mh_flags = 0, mh_archive_number = 0 }, ma_som = 0x0, ma_pfid = { f_seq = 0, f_oid = 0, f_ver = 0 } }, mti_body = 0x0, mti_object = 0x0, mti_dlm_req = 0xffff8804429d80d8, mti_has_trans = 0, mti_no_need_trans = 0, mti_cross_ref = 0, mti_opdata = 33554475, mti_rr = { rr_opcode = REINT_OPEN, rr_handle = 0xffff8804429d8190, rr_fid1 = 0xffff8804429d8170, rr_fid2 = 0xffff8804429d8180, rr_name = 0xffff8804429d81d0 "FICHIERprot_01455000.MpiIO", rr_namelen = 26, rr_tgt = 0x0, rr_tgtlen = 0, rr_eadata = 0x0, rr_eadatalen = 0, rr_logcookielen = 0, rr_logcookies = 0x0, rr_flags = 0 }, mti_mos = 0x0, mti_ver = {128854322096, 1, 1, 128853780336}, mti_spec = { u = { sp_symname = 0x0, sp_pfid = 0x0, sp_ea = { fid = 0x0, eadata = 0xffff8804429d81f0, eadatalen = 96 } }, no_create = 0, sp_cr_flags = 570425345, sp_cr_lookup = 0, sp_cr_mode = MDL_CW, sp_ck_split = 1, sp_feat = 0xffffffffa0681440 }, mti_tmp_fid1 = { f_seq = 8939110390, f_oid = 87594, f_ver = 0 }, mti_tmp_fid2 = { f_seq = 8943489550, f_oid = 273, f_ver = 0 }, mti_policy = { l_extent = { start = 5, end = 0, gid = 0 }, l_flock = { start = 5, end = 0, owner = 0, blocking_owner = 0, blocking_export = 0x0, pid = 0 }, l_inodebits = { bits = 5 } }, mti_res_id = { name = {8939110390, 87594, 0, 0} }, mti_u = { uuid = {{ uuid = "S?000\000\000\000\000\000\000\020\000\000\000\000\000\000\030\200'\v\000\000\000\000ﾏ?.\n\000\000\000\000U?b\n\000\000\000" }, { uuid = "\000\000ﾀ\035\000\000\000\000\223ﾋﾀ\030\000\000\000\000VｸﾈBﾈｱGｭ?\000\000\000\000\000\000\000\000\000\000\000\000\000\000" }}, ns_name = "S?000\000\000\000\000\000\000\020\000\000\000\000\000\000\030\200'\v\000\000\000\000ﾏ?.\n\000\000\000\000U?b\n\000\000\000\000\000\000ﾀ\035\000\000\000", bufs = { lcfg_buf = {0xef53, 0x1000, 0xb278018, 0xa2effcf, 0xa08eb55, 0x1dc00000, 0x18c0cb93, 0xad47b1c842c8b856}, lcfg_buflen = {255, 0, 0, 0, 0, 0, 0, 0}, lcfg_bufcount = 0 }, ksfs = { f_type = 61267, f_bsize = 4096, f_blocks = 187138072, f_bfree = 170852303, f_bavail = 168356693, f_files = 499122176, f_ffree = 415288211, f_fsid = { val = {1120450646, -1387810360} }, f_namelen = 255, f_frsize = 0, f_flags = 0, f_spare = {0, 0, 0, 0} }, rdpg = { mti_rdpg = { rp_hash = 61267, rp_count = 4096, rp_npages = 0, rp_attrs = 187138072, rp_pages = 0xa2effcf }, mti_wait_info = { lwi_timeout = 168356693, lwi_interval = 499122176, lwi_allow_intr = 415288211, lwi_on_timeout = 0xad47b1c842c8b856, lwi_on_signal = 0xff <__mod_ahci_em_messages758+31>, lwi_cb_data = 0x0 } }, som = { attr = { ma_valid = 61267, ma_need = 4096, ma_attr_flags = 187138072, ma_attr = { la_size = 170852303, la_mtime = 168356693, la_atime = 499122176, la_ctime = 415288211, la_blocks = 12486143965580015702, la_mode = 255, la_uid = 0, la_gid = 0, la_flags = 0, la_nlink = 0, la_blkbits = 0, la_blksize = 0, la_rdev = 0, la_valid = 0 }, ma_lmm = 0x0, ma_lmm_size = 0, ma_lmv = 0x0, ma_lmv_size = 0, ma_acl = 0x0, ma_acl_size = 0, ma_cookie = 0x0, ma_cookie_size = 0, ma_capa = 0x0, ma_hsm = { mh_flags = 0, mh_archive_number = 0 }, ma_som = 0x0, ma_pfid = { f_seq = 0, f_oid = 0, f_ver = 0 } }, data = { msd_ioepoch = 0, msd_size = 0, msd_blocks = 0, msd_mountid = 0 } } }, mti_ioepoch = 0x0, mti_replayepoch = 0, mti_lsd = { lsd_uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", lsd_last_transno = 0, lsd_compat14 = 0, lsd_mount_count = 0, lsd_feature_compat = 0, lsd_feature_rocompat = 0, lsd_feature_incompat = 0, lsd_server_size = 0, lsd_client_start = 0, lsd_client_size = 0, lsd_subdir_count = 0, lsd_catalog_oid = 0, lsd_catalog_ogen = 0, lsd_peeruuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", lsd_ost_index = 0, lsd_mdt_index = 0, lsd_start_epoch = 0, lsd_trans_table = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, lsd_trans_table_time = 0, lsd_expire_intervals = 0, lsd_padding = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" }, mti_lcd = { lcd_uuid = "56004b1a-285e-f3fc-bb6e-584873a84c29\000\000\000", lcd_last_transno = 128854336612, lcd_last_xid = 1428762237805967, lcd_last_result = 0, lcd_last_data = 33554475, lcd_last_close_transno = 128854336611, lcd_last_close_xid = 1428762237805969, lcd_last_close_result = 0, lcd_last_close_data = 0, lcd_pre_versions = {0, 0, 0, 0}, lcd_last_epoch = 0, lcd_first_epoch = 0, lcd_padding = 0xffff880462ff0ea8 "" }, mti_off = 0, mti_txn_param = { tp_credits = 67, tp_sync = 0 }, mti_buf = { lb_buf = 0xffff880f91891258, lb_len = 260 }, mti_capa_key = { lk_seq = 0, lk_keyid = 0, lk_padding = 0, lk_key = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" }, mti_name = { ln_name = 0xffff8804429d81d0 "FICHIERprot_01455000.MpiIO", ln_namelen = 26 }, mti_tmp_attr = { ma_valid = 0, ma_need = 0, ma_attr_flags = 0, ma_attr = { la_size = 0, la_mtime = 0, la_atime = 0, la_ctime = 0, la_blocks = 0, la_mode = 0, la_uid = 0, la_gid = 0, la_flags = 0, la_nlink = 0, la_blkbits = 0, la_blksize = 0, la_rdev = 0, la_valid = 0 }, ma_lmm = 0x0, ma_lmm_size = 0, ma_lmv = 0x0, ma_lmv_size = 0, ma_acl = 0x0, ma_acl_size = 0, ma_cookie = 0x0, ma_cookie_size = 0, ma_capa = 0x0, ma_hsm = { mh_flags = 0, mh_archive_number = 0 }, ma_som = 0x0, ma_pfid = { f_seq = 0, f_oid = 0, f_ver = 0 } } } crash> crash> struct mdt_thread_info ffff880462ff0800 | grep uid la_uid = 2078, [root@lascaux1 bullxscs4] # grep 2078 /etc/passwd user:*:2078:3607::/home/user:/bin/csh crash> struct mdt_thread_info.mti_pill ffff880462ff0800 mti_pill = 0xffff880d9abcab70, crash> struct req_capsule 0xffff880d9abcab70 struct req_capsule { rc_req = 0xffff880d9abca800, rc_fmt = 0xffffffffa0805fe0, rc_loc = RCL_SERVER, rc_area = {{4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295}, {152, 112, 216, 3888, 260, 120, 120, 4294967295, 4294967295}} } crash> struct ptlrpc_request 0xffff880d9abca800 struct ptlrpc_request { rq_type = 0, rq_list = { next = 0xffff880d9abca808, prev = 0xffff880d9abca808 }, rq_timed_list = { next = 0xffff88044484d350, prev = 0xffff88044484d350 }, rq_history_list = { next = 0xffff880f8f948028, prev = 0xffff880442907508 }, rq_exp_list = { next = 0x0, prev = 0x0 }, rq_ops = 0x0, rq_history_seq = 9986122, rq_at_index = 309, rq_status = -66, rq_lock = { raw_lock = { slock = 131074 } }, rq_intr = 0, rq_replied = 0, rq_err = 0, rq_timedout = 0, rq_resend = 0, rq_restart = 0, rq_replay = 0, rq_no_resend = 0, rq_waiting = 0, rq_receiving_reply = 0, rq_no_delay = 0, rq_net_err = 0, rq_wait_ctx = 0, rq_early = 0, rq_must_unlink = 0, rq_fake = 0, rq_memalloc = 0, rq_packed_final = 1, rq_hp = 0, rq_at_linked = 1, rq_reply_truncate = 0, rq_committed = 0, rq_invalid_rqset = 0, rq_generation_set = 0, rq_no_retry_einprogress = 0, rq_nr_resend = 0, rq_phase = 3955285507, rq_next_phase = 0, rq_refcount = { counter = 1 }, rq_svc_thread = 0xffff8802c013f540, rq_request_portal = 0, rq_reply_portal = 0, rq_nob_received = 0, rq_reqlen = 592, rq_reqmsg = 0xffff8804429d8000, rq_replen = 4936, rq_repmsg = 0xffff880f91890108, rq_transno = 128854336612, rq_xid = 1428762237805967, rq_replay_list = { next = 0x0, prev = 0x0 }, rq_cli_ctx = 0x0, rq_svc_ctx = 0xffffffffa0811010, rq_ctx_chain = { next = 0x0, prev = 0x0 }, rq_flvr = { sf_rpc = 0, sf_flags = 0, u_rpc = {}, u_bulk = { hash = { hash_alg = 0 '\000' } } }, rq_sp_from = LUSTRE_SP_CLI, rq_ctx_init = 0, rq_ctx_fini = 0, rq_bulk_read = 0, rq_bulk_write = 0, rq_auth_gss = 0, rq_auth_remote = 0, rq_auth_usr_root = 0, rq_auth_usr_mdt = 0, rq_auth_usr_ost = 0, rq_pack_udesc = 0, rq_pack_bulk = 0, rq_no_reply = 0, rq_pill_init = 1, rq_auth_uid = 4294967295, rq_auth_mapped_uid = 4294967295, rq_user_desc = 0x0, rq_reply_off = 0, rq_reqbuf = 0xffff8804429d8000, rq_reqbuf_len = 0, rq_reqdata_len = 592, rq_repbuf = 0x0, rq_repbuf_len = 0, rq_repdata = 0x0, rq_repdata_len = 0, rq_clrbuf = 0x0, rq_clrbuf_len = 0, rq_clrdata_len = 0, rq_req_swab_mask = 0, rq_rep_swab_mask = 0, rq_import_generation = 0, rq_send_state = 0, rq_early_count = 0, rq_req_md_h = { cookie = 0 }, rq_req_cbid = { cbid_fn = 0, cbid_arg = 0x0 }, rq_delay_limit = 0, rq_queued_time = 0, rq_arrival_time = { tv_sec = 1362582012, tv_usec = 47957 }, rq_reply_state = 0xffff880f91890000, rq_rqbd = 0xffff880462c47800, rq_reply_md_h = { cookie = 0 }, rq_reply_waitq = { lock = { raw_lock = { slock = 0 } }, task_list = { next = 0x0, prev = 0x0 } }, rq_reply_cbid = { cbid_fn = 0, cbid_arg = 0x0 }, rq_self = 1407422302593066, rq_peer = { nid = 1407422302596115, pid = 12345 }, rq_export = 0xffff8806065c3800, rq_import = 0x0, rq_replay_cb = 0, rq_commit_cb = 0, rq_cb_data = 0x0, rq_bulk = 0x0, rq_sent = 0, rq_real_sent = 0, rq_deadline = 1362582029, rq_reply_deadline = 0, rq_bulk_deadline = 0, rq_timeout = 0, rq_set_chain = { next = 0x0, prev = 0x0 }, rq_set_waitq = { lock = { raw_lock = { slock = 0 } }, task_list = { next = 0x0, prev = 0x0 } }, rq_set = 0x0, rq_interpret_reply = 0, rq_async_args = { pointer_arg = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, space = {0, 0, 0, 0, 0, 0} }, rq_pool = 0x0, rq_session = { lc_tags = 2684354576, lc_thread = 0xffff8802c013f540, lc_value = 0xffff880f7e8411c0, lc_state = LCS_ENTERED, lc_remember = { next = 0xffff88020c22feb0, prev = 0xffffffffa0657790 }, lc_version = 37, lc_cookie = 5 }, rq_recov_session = { lc_tags = 0, lc_thread = 0x0, lc_value = 0x0, lc_state = 0, lc_remember = { next = 0x0, prev = 0x0 }, lc_version = 0, lc_cookie = 0 }, rq_pill = { rc_req = 0xffff880d9abca800, rc_fmt = 0xffffffffa0805fe0, rc_loc = RCL_SERVER, rc_area = {{4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295, 4294967295}, {152, 112, 216, 3888, 260, 120, 120, 4294967295, 4294967295}} } } crash> crash> struct ptlrpc_request.rq_status 0xffff880d9abca800 rq_status = -66, #define EREMOTE 66 /* Object is remote */ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc) { struct mdt_device *mdt = info->mti_mdt; struct ptlrpc_request *req = mdt_info_req(info); struct mdt_object *parent; struct mdt_object *child; struct mdt_lock_handle *lh; .......... out: if (result && result != -EREMOTE) lustre_msg_set_transno(req->rq_repmsg, 0); return result; } int mdt_get_disposition(struct ldlm_reply *rep, int flag) { if (!rep) return 0; return (rep->lock_policy_res1 & flag); } ldlm_rep = req_capsule_server_get(pill, &RMF_DLM_REP); /** * Trivial wrapper around __req_capsule_get(), that returns the PTLRPC reply * buffer corresponding to the given RMF (\a field) of a \a pill. */ void *req_capsule_server_get(struct req_capsule *pill, const struct req_msg_field *field) { return __req_capsule_get(pill, field, RCL_SERVER, NULL, 0); } struct tg_export_data *ted = &req->rq_export->exp_target_data; crash> struct ptlrpc_request.rq_export 0xffff880d9abca800 rq_export = 0xffff8806065c3800, crash> p (*(struct ptlrpc_request *)0xffff880d9abca800).rq_export $3 = (struct obd_export *) 0xffff8806065c3800 crash> struct obd_export.u 0xffff8806065c3800 u = { eu_target_data = { crash> p (*(struct obd_export *)0xffff8806065c3800).u.eu_target_data $5 = { ted_lcd_lock = { lock = { raw_lock = { slock = 4194368 } }, count = 1, wait_list = { next = 0xffff8806065c39f0, prev = 0xffff8806065c39f0 } }, ted_lcd = 0xffff88060b574cc0, ted_lr_off = 207360, ted_lr_idx = 1556 } crash> struct lsd_client_data 0xffff88060b574cc0 struct lsd_client_data { lcd_uuid = "56004b1a-285e-f3fc-bb6e-584873a84c29\000\000\000", lcd_last_transno = 128854336612, lcd_last_xid = 1428762237805967, lcd_last_result = 0, lcd_last_data = 33554475, <--- 0x0200002B #define DISP_IT_EXECD 0x00000001 #define DISP_LOOKUP_EXECD 0x00000002 #define DISP_LOOKUP_POS 0x00000008 #define DISP_OPEN_OPEN 0x00000020 #define DISP_OPEN_LOCK 0x02000000 lcd_last_close_transno = 128854336611, lcd_last_close_xid = 1428762237805969, lcd_last_close_result = 0, lcd_last_close_data = 0, lcd_pre_versions = {0, 0, 0, 0}, lcd_last_epoch = 0, lcd_first_epoch = 0, lcd_padding = 0xffff88060b574d40 "ﾀ\212ｮ\200\a\210??" } crash> struct ptlrpc_request.rq_transno 0xffff880d9abca800 rq_transno = 128854336612, void mdt_set_disposition(struct mdt_thread_info *info, struct ldlm_reply *rep, int flag) { if (info) info->mti_opdata |= flag; if (rep) rep->lock_policy_res1 |= flag; } int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc) { struct mdt_device *mdt = info->mti_mdt; struct ptlrpc_request *req = mdt_info_req(info); struct mdt_object *parent; struct mdt_object *child; struct mdt_lock_handle *lh; struct ldlm_reply *ldlm_rep; struct mdt_body *repbody; struct lu_fid *child_fid = &info->mti_tmp_fid1; struct md_attr *ma = &info->mti_attr; __u64 create_flags = info->mti_spec.sp_cr_flags; struct mdt_reint_record *rr = &info->mti_rr; struct lu_name *lname; int result, rc; int created = 0; __u32 msg_flags; ENTRY; OBD_FAIL_TIMEOUT_ORSET(OBD_FAIL_MDS_PAUSE_OPEN, OBD_FAIL_ONCE, (obd_timeout + 1) / 4); mdt_counter_incr(req->rq_export, LPROC_MDT_OPEN); repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); ma->ma_lmm = req_capsule_server_get(info->mti_pill, &RMF_MDT_MD); ma->ma_lmm_size = req_capsule_get_size(info->mti_pill, &RMF_MDT_MD, RCL_SERVER); ma->ma_need = MA_INODE; if (ma->ma_lmm_size > 0) ma->ma_need |= MA_LOV; ma->ma_valid = 0; LASSERT(info->mti_pill->rc_fmt == &RQF_LDLM_INTENT_OPEN); ldlm_rep = req_capsule_server_get(info->mti_pill, &RMF_DLM_REP); if (unlikely(create_flags & MDS_OPEN_JOIN_FILE)) { CERROR("file join is not supported anymore.\n"); GOTO(out, result = err_serious(-EOPNOTSUPP)); } msg_flags = lustre_msg_get_flags(req->rq_reqmsg); if ((create_flags & (MDS_OPEN_HAS_EA | MDS_OPEN_HAS_OBJS)) && info->mti_spec.u.sp_ea.eadata == NULL) GOTO(out, result = err_serious(-EINVAL)); CDEBUG(D_INODE, "I am going to open "DFID"/(%s->"DFID") " "cr_flag="LPO64" mode=0%06o msg_flag=0x%x\n", PFID(rr->rr_fid1), rr->rr_name, PFID(rr->rr_fid2), create_flags, ma->ma_attr.la_mode, msg_flags); if (req_is_replay(req) || (req->rq_export->exp_libclient && create_flags&MDS_OPEN_HAS_EA)) { /* This is a replay request or from liblustre with ea. */ result = mdt_open_by_fid(info, ldlm_rep); if (result != -ENOENT) { if (req->rq_export->exp_libclient && create_flags&MDS_OPEN_HAS_EA) GOTO(out, result = 0); GOTO(out, result); } /* * We didn't find the correct object, so we need to re-create it * via a regular replay. */ if (!(create_flags & MDS_OPEN_CREAT)) { DEBUG_REQ(D_ERROR, req, "OPEN & CREAT not in open replay."); GOTO(out, result = -EFAULT); } CDEBUG(D_INFO, "Open replay did find object, continue as " "regular open\n"); } else if (rr->rr_namelen == 0 && !info->mti_cross_ref && create_flags & MDS_OPEN_LOCK) { result = mdt_open_anon_by_fid(info, ldlm_rep, lhc); GOTO(out, result); } if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OPEN_PACK)) GOTO(out, result = err_serious(-ENOMEM)); mdt_set_disposition(info, ldlm_rep, (DISP_IT_EXECD | DISP_LOOKUP_EXECD)); if (info->mti_cross_ref) { /* This is cross-ref open */ mdt_set_disposition(info, ldlm_rep, DISP_LOOKUP_POS); result = mdt_cross_open(info, rr->rr_fid1, ldlm_rep, create_flags); GOTO(out, result); } lh = &info->mti_lh[MDT_LH_PARENT]; mdt_lock_pdo_init(lh, (create_flags & MDS_OPEN_CREAT) ? LCK_PW : LCK_PR, rr->rr_name, rr->rr_namelen); parent = mdt_object_find_lock(info, rr->rr_fid1, lh, MDS_INODELOCK_UPDATE); if (IS_ERR(parent)) GOTO(out, result = PTR_ERR(parent)); /* get and check version of parent */ result = mdt_version_get_check(info, parent, 0); if (result) GOTO(out_parent, result); fid_zero(child_fid); lname = mdt_name(info->mti_env, (char *)rr->rr_name, rr->rr_namelen); result = mdo_lookup(info->mti_env, mdt_object_child(parent), lname, child_fid, &info->mti_spec); LASSERTF(ergo(result == 0, fid_is_sane(child_fid)), "looking for "DFID"/%s, result fid="DFID"\n", PFID(mdt_object_fid(parent)), rr->rr_name, PFID(child_fid)); if (result != 0 && result != -ENOENT && result != -ESTALE) GOTO(out_parent, result); if (result == -ENOENT || result == -ESTALE) { mdt_set_disposition(info, ldlm_rep, DISP_LOOKUP_NEG); if (result == -ESTALE) { /* * -ESTALE means the parent is a dead(unlinked) dir, so * it should return -ENOENT to in accordance with the * original mds implementaion. */ GOTO(out_parent, result = -ENOENT); } if (!(create_flags & MDS_OPEN_CREAT)) GOTO(out_parent, result); *child_fid = *info->mti_rr.rr_fid2; LASSERTF(fid_is_sane(child_fid), "fid="DFID"\n", PFID(child_fid)); } else { /* * Check for O_EXCL is moved to the mdt_finish_open(), we need to * return FID back in that case. */ mdt_set_disposition(info, ldlm_rep, DISP_LOOKUP_POS); } child = mdt_object_find(info->mti_env, mdt, child_fid); if (IS_ERR(child)) GOTO(out_parent, result = PTR_ERR(child)); /** check version of child */ rc = mdt_version_get_check(info, child, 1); if (rc) GOTO(out_child, result = rc); mdt_set_capainfo(info, 1, child_fid, BYPASS_CAPA); if (result == -ENOENT) { if (mdt_object_obf(parent)) GOTO(out_child, result = -EPERM); /* save versions in reply */ mdt_version_get_save(info, parent, 0); mdt_version_get_save(info, child, 1); /* version of child will be changed */ info->mti_mos = child; /* Not found and with MDS_OPEN_CREAT: let's create it. */ mdt_set_disposition(info, ldlm_rep, DISP_OPEN_CREATE); /* Let lower layers know what is lock mode on directory. */ info->mti_spec.sp_cr_mode = mdt_dlm_mode2mdl_mode(lh->mlh_pdo_mode); /* * Do not perform lookup sanity check. We know that name does * not exist. */ info->mti_spec.sp_cr_lookup = 0; info->mti_spec.sp_feat = &dt_directory_features; result = mdo_create(info->mti_env, mdt_object_child(parent), lname, mdt_object_child(child), &info->mti_spec, &info->mti_attr); if (result == -ERESTART) { mdt_clear_disposition(info, ldlm_rep, DISP_OPEN_CREATE); GOTO(out_child, result); } else { if (result != 0) GOTO(out_child, result); } created = 1; } else { /* We have to get attr & lov ea for this object */ result = mo_attr_get(info->mti_env, mdt_object_child(child), ma); /* * The object is on remote node, return its FID for remote open. */ if (result == -EREMOTE) { /* * Check if this lock already was sent to client and * this is resent case. For resent case do not take lock * again, use what is already granted. */ LASSERT(lhc != NULL); if (lustre_handle_is_used(&lhc->mlh_reg_lh)) { struct ldlm_lock *lock; LASSERT(msg_flags & MSG_RESENT); lock = ldlm_handle2lock(&lhc->mlh_reg_lh); if (!lock) { CERROR("Invalid lock handle "LPX64"\n", lhc->mlh_reg_lh.cookie); LBUG(); } LASSERT(fid_res_name_eq(mdt_object_fid(child), &lock->l_resource->lr_name)); LDLM_LOCK_PUT(lock); rc = 0; } else { mdt_lock_handle_init(lhc); mdt_lock_reg_init(lhc, LCK_PR); rc = mdt_object_lock(info, child, lhc, MDS_INODELOCK_LOOKUP, MDT_CROSS_LOCK); } repbody->fid1 = *mdt_object_fid(child); repbody->valid |= (OBD_MD_FLID | OBD_MD_MDS); if (rc != 0) result = rc; GOTO(out_child, result); } } LASSERT(!lustre_handle_is_used(&lhc->mlh_reg_lh)); /* get openlock if this is not replay and if a client requested it */ if (!req_is_replay(req) && create_flags & MDS_OPEN_LOCK) { ldlm_mode_t lm; if (create_flags & FMODE_WRITE) lm = LCK_CW; else if (create_flags & MDS_FMODE_EXEC) lm = LCK_PR; else lm = LCK_CR; mdt_lock_handle_init(lhc); mdt_lock_reg_init(lhc, lm); rc = mdt_object_lock(info, child, lhc, MDS_INODELOCK_LOOKUP | MDS_INODELOCK_OPEN, MDT_CROSS_LOCK); if (rc) { result = rc; GOTO(out_child, result); } else { result = -EREMOTE; mdt_set_disposition(info, ldlm_rep, DISP_OPEN_LOCK); } } } /* Try to open it now. */ rc = mdt_finish_open(info, parent, child, create_flags, created, ldlm_rep); if (rc) { result = rc; if (lustre_handle_is_used(&lhc->mlh_reg_lh)) /* openlock was acquired and mdt_finish_open failed - drop the openlock */ mdt_object_unlock(info, child, lhc, 1); if (created) { ma->ma_need = 0; ma->ma_valid = 0; ma->ma_cookie_size = 0; info->mti_no_need_trans = 1; rc = mdo_unlink(info->mti_env, mdt_object_child(parent), mdt_object_child(child), lname, &info->mti_attr); if (rc != 0) CERROR("Error in cleanup of open\n"); } } EXIT; out_child: mdt_object_put(info->mti_env, child); out_parent: mdt_object_unlock_put(info, parent, lh, result || !created); out: if (result && result != -EREMOTE) lustre_msg_set_transno(req->rq_repmsg, 0); return result; }