[sanity]$ sys_stat .lustre/fid dev 743766374 ino 144115188109410306 mode 16877 nlink 2 uid 0 gid 0 rdev 0 size 4096 blksize 4096 blocks 8 atime 1345575550 mtime 1345575463 ctime 1345575463 [sanity]$ sys_chmod .lustre/fid 0666 sys_chmod: cannot change mode of `.lustre/fid' to 00666: Operation not permitted [sanity]$ sys_chown .lustre/fid 7 7 sys_chown: cannot change owner of `.lustre/fid' to uid 7, gid 7: Operation not permitted [sanity]$ sys_creat .lustre/fid/XXX sys_creat: cannot create `.lustre/fid/XXX' with mode 0666: Invalid argument [sanity]$ sys_creat .lustre/fid/[0x1:0x2:0x0] Lustre: ctl-lustre-MDT0000: super-sequence allocation rc = 0 [0x0000000200000400-0x0000000240000400):0:mdt Lustre: 7696:0:(mdd_device.c:872:obf_lookup()) bad FID format [XXX], should be [0x1:0x2:0x0] LustreError: 8033:0:(dir.c:437:ll_get_dir_page()) read cache page: [0x200000002:0x2:0x0] at 0: rc -1 LustreError: 8033:0:(dir.c:613:ll_readdir()) error reading dir [0x200000002:0x2:0x0] at 0: rc -1 LustreError: 7696:0:(osd_compat.c:381:osd_compat_objid_lookup()) ASSERTION( map ) failed: LustreError: 7696:0:(osd_compat.c:381:osd_compat_objid_lookup()) LBUG Pid: 7696, comm: mdt00_002 Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] lbug_with_loc+0x47/0xb0 [libcfs] [] osd_compat_objid_lookup+0x479/0x520 [osd_ldiskfs] [] osd_oi_lookup+0x55/0xc0 [osd_ldiskfs] [] osd_object_init+0x3b2/0xd20 [osd_ldiskfs] [] ? dt_object_init+0xe/0x10 [obdclass] [] lu_object_alloc+0xd5/0x310 [obdclass] [] ? htable_lookup+0x108/0x1c0 [obdclass] [] lu_object_find_at+0x201/0x450 [obdclass] [] ? vsscanf+0x617/0x7c0 [] lu_object_find_slice+0x1f/0x80 [obdclass] [] mdd_object_find+0x10/0x70 [mdd] [] obf_lookup+0xd6/0x270 [mdd] [] cml_lookup+0x66/0x1b0 [cmm] [] ? mdt_version_get_check+0x47/0xe0 [mdt] [] mdt_reint_open+0x7a7/0x18a0 [mdt] [] ? md_ucred+0x1e/0x60 [mdd] [] ? mdt_ucred+0x15/0x20 [mdt] [] ? mdt_root_squash+0x2c/0x3e0 [mdt] [] mdt_reint_rec+0x41/0xe0 [mdt] [] mdt_reint_internal+0x50a/0x810 [mdt] [] mdt_intent_reint+0x1ed/0x500 [mdt] [] mdt_intent_policy+0x371/0x6a0 [mdt] [] ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] [] ldlm_handle_enqueue0+0x48f/0xf70 [ptlrpc] [] mdt_enqueue+0x46/0x130 [mdt] [] mdt_handle_common+0x922/0x1740 [mdt] [] mdt_regular_handle+0x15/0x20 [mdt] [] ptlrpc_server_handle_request+0x40d/0xea0 [ptlrpc] [] ? cfs_timer_arm+0xe/0x10 [libcfs] [] ? ptlrpc_wait_event+0xa7/0x2a0 [ptlrpc] [] ? __wake_up+0x53/0x70 [] ptlrpc_main+0xb59/0x1860 [ptlrpc] [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] child_rip+0xa/0x20 [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? child_rip+0x0/0x20 Kernel panic - not syncing: LBUG Pid: 7696, comm: mdt00_002 Not tainted 2.6.32-279.5.1.el6.x86_64 #1 Call Trace: [] ? panic+0xa0/0x168 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] [] ? osd_compat_objid_lookup+0x479/0x520 [osd_ldiskfs] [] ? osd_oi_lookup+0x55/0xc0 [osd_ldiskfs] [] ? osd_object_init+0x3b2/0xd20 [osd_ldiskfs] [] ? dt_object_init+0xe/0x10 [obdclass] [] ? lu_object_alloc+0xd5/0x310 [obdclass] [] ? htable_lookup+0x108/0x1c0 [obdclass] [] ? lu_object_find_at+0x201/0x450 [obdclass] [] ? vsscanf+0x617/0x7c0 [] ? lu_object_find_slice+0x1f/0x80 [obdclass] [] ? mdd_object_find+0x10/0x70 [mdd] [] ? obf_lookup+0xd6/0x270 [mdd] [] ? cml_lookup+0x66/0x1b0 [cmm] [] ? mdt_version_get_check+0x47/0xe0 [mdt] [] ? mdt_reint_open+0x7a7/0x18a0 [mdt] [] ? md_ucred+0x1e/0x60 [mdd] [] ? mdt_ucred+0x15/0x20 [mdt] [] ? mdt_root_squash+0x2c/0x3e0 [mdt] [] ? mdt_reint_rec+0x41/0xe0 [mdt] [] ? mdt_reint_internal+0x50a/0x810 [mdt] [] ? mdt_intent_reint+0x1ed/0x500 [mdt] [] ? mdt_intent_policy+0x371/0x6a0 [mdt] [] ? ldlm_lock_enqueue+0x361/0x8f0 [ptlrpc] [] ? ldlm_handle_enqueue0+0x48f/0xf70 [ptlrpc] [] ? mdt_enqueue+0x46/0x130 [mdt] [] ? mdt_handle_common+0x922/0x1740 [mdt] [] ? mdt_regular_handle+0x15/0x20 [mdt] [] ? ptlrpc_server_handle_request+0x40d/0xea0 [ptlrpc] [] ? cfs_timer_arm+0xe/0x10 [libcfs] [] ? ptlrpc_wait_event+0xa7/0x2a0 [ptlrpc] [] ? __wake_up+0x53/0x70 [] ? ptlrpc_main+0xb59/0x1860 [ptlrpc] [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? child_rip+0xa/0x20 [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? child_rip+0x0/0x20 [sanity]$ sys_getdents .lustre/fid sys_getdents: cannot read directory `.lustre/fid': Operation not permitted [sanity]$ sys_lea .lustre/fid sys_lea: cannot retreive value for attribute `trusted.link' of `.lustre/fid': Operation not permitted [sanity]$ sys_link .lustre/fid fud sys_link: cannot link `fud' to `.lustre/fid': Permission denied [sanity]$ sys_listxattr .lustre/fid sys_listxattr: cannot retreive extended attribute list for `.lustre/fid': No data available [sanity]$ sys_lma .lustre/fid sys_lma: cannot retreive value for attribute `trusted.lma' of `.lustre/fid': Operation not permitted [sanity]$ sys_lstat .lustre/fid [sanity]$ sys_mkdir .lustre/fid sys_mkdir: cannot mkdir `.lustre/fid' with mode 0777: File exists [sanity]$ sys_mkdir .lustre/fud sys_mkdir: cannot mkdir `.lustre/fud' with mode 0777: Permission denied [sanity]$ sys_mkdir .lustre/fid/fud sys_mkdir: cannot mkdir `.lustre/fid/fud' with mode 0777: Permission denied [sanity]$ sys_mkdir .lustre/fid/[0x1:0x2:0x0] sys_mkdir: cannot mkdir `.lustre/fid/[0x1:0x2:0x0]' with mode 0777: Permission denied [sanity]$ sys_mkdir .lustre/fid/[0x2:0x1:0x0] sys_mkdir: cannot mkdir `.lustre/fid/[0x2:0x1:0x0]' with mode 0777: Permission denied [sanity]$ sys_mknod .lustre/fid/XXX r 0 0 cannot create device `.lustre/fid/XXX' mode 100666, maj 0, min 0: Permission denied [sanity]$ sys_mknod .lustre/fid/[0x0:0x0:0x0] r 0 0 cannot create device `.lustre/fid/[0x0:0x0:0x0]' mode 100666, maj 0, min 0: Permission denied [sanity]$ sys_mknod .lustre/fid/[0x1:0x2:0x0] r 0 0 cannot create device `.lustre/fid/[0x1:0x2:0x0]' mode 100666, maj 0, min 0: Permission denied [sanity]$ sys_mknod .lustre/XXX r 0 0 cannot create device `.lustre/XXX' mode 100666, maj 0, min 0: Permission denied [sanity]$ sys_path2fid .lustre/fid/XXX cannot open `.lustre/fid/XXX' with flags 0, mode 0666: Invalid argument [sanity]$ sys_path2fid .lustre/fid/[0x0:0x0:0x0] cannot open `.lustre/fid/[0x0:0x0:0x0]' with flags 0, mode 0666: Invalid argument [sanity]$ sys_path2fid .lustre/fid/[0x1:0x2:0x0] LustreError: 2615:0:(osd_compat.c:381:osd_compat_objid_lookup()) ASSERTION( map ) failed: LustreError: 2615:0:(osd_compat.c:381:osd_compat_objid_lookup()) LBUG ... [sanity]$ sys_readlink .lustre/fid sys_readlink: cannot read link `.lustre/fid': Invalid argument [sanity]$ sys_rename .lustre/fid .lustre/fud cannot rename `.lustre/fid' to `.lustre/fud': Permission denied [sanity]$ sys_rmdir .lustre/fid cannot delete directory `.lustre/fid': Permission denied [sanity]$ sys_stat .lustre/fid dev 743766374 ino 144115188109410306 mode 16877 nlink 2 uid 0 gid 0 rdev 0 size 4096 blksize 4096 blocks 8 atime 1345577050 mtime 1345577050 ctime 1345577050 [sanity]$ sys_symlink barf .lustre/fid/LINK cannot create symbolic link `.lustre/fid/LINK' containing `barf': Permission denied [sanity]$ sys_symlink barf .lustre/LINK cannot create symbolic link `.lustre/LINK' containing `barf': Permission denied [sanity]$ sys_truncate .lustre/fid cannot truncate `.lustre/fid' to length 0: Is a directory [sanity]$ sys_unlink .lustre/fid cannot unlink `.lustre/fid': Permission denied sanity]$ sys_mkdir sanity/XXX [sanity]$ sys_rename sanity/XXX .lustre/fid cannot rename `sanity/XXX' to `.lustre/fid': Permission denied [sanity]$ sys_rename sanity/XXX .lustre/fid/XXX cannot rename `sanity/XXX' to `.lustre/fid/XXX': Invalid argument [sanity]$ sys_rename .lustre/fid sanity/XXX cannot rename `.lustre/fid' to `sanity/XXX': Permission denied [root]# sys_chown .lustre/fid 7 7 sys_chown: cannot change owner of `.lustre/fid' to uid 7, gid 7: Operation not permitted [root]# sys_chmod -- .lustre/fid -1 sys_chmod: cannot change mode of `.lustre/fid' to 37777777777: Operation not permitted [root]# sys_chmod -- .lustre/fid 0777 sys_chmod: cannot change mode of `.lustre/fid' to 00777: Operation not permitted [root]# sys_creat .lustre/fud [root]# sys_getdents .lustre d 25003 `..' d 144115188109410305 `.' r 144115205255725057 `fud' [root]# sys_stat .lustre/fud cannot stat `.lustre/fud': No such file or directory [root]# sys_creat .lustre/fid/XXX sys_creat: cannot create `.lustre/fid/XXX' with mode 0666: Invalid argument [root]# sys_creat .lustre/fid/[0x0:0x0:0x0] sys_creat: cannot create `.lustre/fid/[0x0:0x0:0x0]' with mode 0666: Invalid argument [root]# sys_creat .lustre/fid/[0x1:0x2:0x0] LustreError: 2594:0:(osd_compat.c:381:osd_compat_objid_lookup()) ASSERTION( map ) failed: ... [root]# sys_getdents .lustre/fid sys_getdents: cannot read directory `.lustre/fid': Input/output error [root]# sys_mkdir XXX [root]# sys_rename XXX .lustre/fid BUG: unable to handle kernel NULL pointer dereference at 0000000000000340 IP: [] osd_xattr_get+0x170/0x350 [osd_ldiskfs] PGD 7d04e067 PUD 7b5be067 PMD 0 Oops: 0000 [#1] SMP last sysfs file: /sys/devices/system/cpu/possible CPU 0 Modules linked in: lustre(U) obdfilter(U) ost(U) cmm(U) mdt(U) osd_ldiskfs(U) fsfilt_ldiskfs(U) ldiskfs(U) exportfs mdd(U) mds(U) mgs(U) lquota(U) jbd obdecho(U) mgc(U) lov(U) osc(U) mdc(U) lmv(U) fid(U) fld(U) ptlrpc(U) obdclass(U) lvfs(U) ksocklnd(U) lnet(U) sha512_generic sha256_generic libcfs(U) autofs4 nfs lockd fscache nfs_acl auth_rpcgss sunrpc ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 iptable_filter ip_tables ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables ipv6 dm_mirror dm_region_hash dm_log dm_mod microcode virtio_balloon virtio_net i2c_piix4 i2c_core ext4 mbcache jbd2 virtio_blk pata_acpi ata_generic ata_piix virtio_pci virtio_ring virtio [last unloaded: speedstep_lib] Pid: 2597, comm: mdt00_002 Not tainted 2.6.32-279.5.1.el6.x86_64 #1 Bochs Bochs RIP: 0010:[] [] osd_xattr_get+0x170/0x350 [osd_ldiskfs] RSP: 0018:ffff88006638baa0 EFLAGS: 00010246 RAX: 0000000000000000 RBX: ffffffffffffff30 RCX: 0000000000000000 RDX: ffff88006627c4c0 RSI: ffffffffa04c7a3c RDI: ffffffffa0ac872b RBP: ffff88006638bae0 R08: fffffffffffffffe R09: 00000000fffffffe R10: ffff88007d3323c0 R11: 0000000000000004 R12: ffff88006638bb08 R13: ffff880066328bc0 R14: ffffffffa04c7a2c R15: 0000000000000000 FS: 00007f86e7d33700(0000) GS:ffff880002200000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b CR2: 0000000000000340 CR3: 000000005cd6a000 CR4: 00000000000006f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process mdt00_002 (pid: 2597, threadinfo ffff88006638a000, task ffff88007bc21540) Stack: fffffffffffffffe ffff8800663b0000 ffff880066389000 ffff88007d334980 ffff880066389000 ffff880066389268 000000000000004d ffff88006638bc60 ffff88006638bb10 ffffffffa0497f84 ffff88006638bb08 0000000000000008 Call Trace: [] dt_version_get+0x54/0x190 [obdclass] [] mdt_obj_version_get+0x6f/0x1f0 [mdt] [] mdt_version_get_save+0x76/0xd0 [mdt] [] mdt_reint_rename+0x2174/0x2200 [mdt] [] ? ldlm_blocking_ast+0x0/0x180 [ptlrpc] [] ? ldlm_completion_ast+0x0/0x730 [ptlrpc] [] ? __req_capsule_get+0x176/0x750 [ptlrpc] [] ? lustre_pack_reply_flags+0xb6/0x210 [ptlrpc] [] mdt_reint_rec+0x41/0xe0 [mdt] [] mdt_reint_internal+0x50a/0x810 [mdt] [] mdt_reint+0x44/0xe0 [mdt] [] mdt_handle_common+0x922/0x1740 [mdt] [] mdt_regular_handle+0x15/0x20 [mdt] [] ptlrpc_server_handle_request+0x40d/0xea0 [ptlrpc] [] ? cfs_timer_arm+0xe/0x10 [libcfs] [] ? ptlrpc_wait_event+0xa7/0x2a0 [ptlrpc] [] ? __wake_up+0x53/0x70 [] ptlrpc_main+0xb59/0x1860 [ptlrpc] [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] child_rip+0xa/0x20 [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? ptlrpc_main+0x0/0x1860 [ptlrpc] [] ? child_rip+0x0/0x20 Code: 33 4b 03 00 00 00 00 00 c7 05 21 4b 03 00 02 00 00 00 48 c7 c7 80 c7 ad a0 48 8b 48 40 48 8b 93 10 04 00 00 31 c0 e8 50 f8 87 ff <48> 8b 83 10 04 00 00 49 89 04 24 b8 08 00 00 00 48 8b 5d d8 4c RIP [] osd_xattr_get+0x170/0x350 [osd_ldiskfs] RSP [root]# mkdir XXX [root]# sys_path2fid XXX [0x200000400:0x3:0x0] [root]# sys_rename XXX .lustre/fid/[0x200000400:0x3:0x0] [root]# sys_rename .lustre/fid/[0x200000400:0x3:0x0] XXX [root]# sys_rename .lustre/fid .lustre/fid [root]# sys_path2fid .lustre/fid [0x200000002:0x2:0x0] [root]# sys_rename .lustre/fid .lustre/fid/[0x200000002:0x2:0x0] cannot rename `.lustre/fid' to `.lustre/fid/[0x200000002:0x2:0x0]': Invalid argument [root]# sys_rename .lustre/fid/[0x200000002:0x2:0x0] .lustre/fid cannot rename `.lustre/fid/[0x200000002:0x2:0x0]' to `.lustre/fid': Directory not empty [root]# sys_rename .lustre/fid/[0x200000002:0x2:0x0] .lustre/fid/[0x200000002:0x2:0x0] [root]# sys_mknod .lustre/fid/PIPE f 0 0 cannot create device `.lustre/fid/PIPE' mode 10666, maj 0, min 0: Operation not permitted [root]# sys_mknod .lustre/fid/PIPE r 0 0 cannot create device `.lustre/fid/PIPE' mode 100666, maj 0, min 0: Operation not permitted [root]# sys_mknod .lustre/fid/PIPE d 0 0 invalid type `d' [root]# sys_mknod .lustre/fid/PIPE c 0 0 cannot create device `.lustre/fid/PIPE' mode 20666, maj 0, min 0: Operation not permitted [root]# sys_rename .lustre/fid .lustre cannot rename `.lustre/fid' to `.lustre': Directory not empty [root]# sys_getdents .lustre d 144115188109410305 `.' d 25003 `..' [root]# sys_rename .lustre .lustre/fid cannot rename `.lustre' to `.lustre/fid': Invalid argument [root]# sys_path2fid .lustre [0x61ac:0xbe3e3350:0x0] [root]# sys_rename .lustre .lustre/fid/[0x61ac:0xbe3e3350:0x0] cannot rename `.lustre' to `.lustre/fid/[0x61ac:0xbe3e3350:0x0]': Invalid argument [root]# sys_rename .lustre/fid/[0x61ac:0xbe3e3350:0x0] .lustre/ cannot rename `.lustre/fid/[0x61ac:0xbe3e3350:0x0]' to `.lustre/': Directory not empty [root]# sys_path2fid . [0x61ab:0xbe3e334e:0x0] [root]# sys_rename .lustre .lustre/fid/[0x61ac:0xbe3e3350:0x0] cannot rename `.lustre' to `.lustre/fid/[0x61ac:0xbe3e3350:0x0]': Invalid argument [root]# sys_rename .lustre/fid/[0x61ac:0xbe3e3350:0x0] .lustre/ cannot rename `.lustre/fid/[0x61ac:0xbe3e3350:0x0]' to `.lustre/': Directory not empty [root]# sys_path2fid . [0x61ab:0xbe3e334e:0x0] [root]# sys_rename .lustre/ .lustre/fid/[0x61ab:0xbe3e334e:0x0]/.lustre INFO: task sys_rename:3139 blocked for more than 120 seconds. "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. sys_rename D 0000000000000000 0 3139 1444 0x00000084 ffff88005cd8fcf8 0000000000000086 ffff88005cd8fd08 ffffffff81189a05 0000000000000000 ffffea000144f6b0 ffffffff8100bc0e ffff88005cd8fcf8 ffff880062a79098 ffff88005cd8ffd8 000000000000fb88 ffff880062a79098 Call Trace: [] ? __link_path_walk+0x155/0x1030 [] ? apic_timer_interrupt+0xe/0x20 [] ? mutex_spin_on_owner+0x9b/0xc0 [] __mutex_lock_slowpath+0x13e/0x180 [] mutex_lock+0x2b/0x50 [] lock_rename+0x73/0xe0 [] sys_renameat+0x113/0x260 [] ? mntput_no_expire+0x30/0x110 [] ? __fput+0x1a1/0x210 [] ? remove_vma+0x6e/0x90 [] ? audit_syscall_entry+0x272/0x2a0 [] ? do_page_fault+0x3e/0xa0 [] sys_rename+0x1b/0x20 [] system_call_fastpath+0x16/0x1b [sanity]$ sys_ sys_access sys_creat sys_listxattr sys_mkdirat sys_readlink sys_symlink sys_chdir sys_getdents sys_lma sys_mknod sys_rename sys_truncate sys_chmod sys_lea sys_lstat sys_open sys_rmdir sys_unlink sys_chown sys_link sys_mkdir sys_path2fid sys_stat