# This file describes further investigations according to # SR 28763 and LU-4504. root@uc1n990:/pfs/work2# lfs quota -v -u es_asaramet . Disk quotas for user es_asaramet (uid 900044): Filesystem kbytes quota limit grace files quota limit grace . 2036560 0 0 - 56587 0 0 - pfs2wor2-MDT0000_UUID 35176 - 0 - 56587 - 0 - pfs2wor2-OST0000_UUID 34932 - 0 - - - - - pfs2wor2-OST0001_UUID 8 - 0 - - - - - pfs2wor2-OST0002_UUID 89096 - 0 - - - - - pfs2wor2-OST0003_UUID 36816 - 0 - - - - - pfs2wor2-OST0004_UUID 37672 - 0 - - - - - pfs2wor2-OST0005_UUID 198896 - 0 - - - - - pfs2wor2-OST0006_UUID 36472 - 0 - - - - - pfs2wor2-OST0007_UUID 0 - 0 - - - - - pfs2wor2-OST0008_UUID 58596 - 0 - - - - - pfs2wor2-OST0009_UUID 41052 - 0 - - - - - pfs2wor2-OST000a_UUID 34304 - 0 - - - - - pfs2wor2-OST000b_UUID 35096 - 0 - - - - - pfs2wor2-OST000c_UUID 33584 - 0 - - - - - pfs2wor2-OST000d_UUID 0 - 0 - - - - - pfs2wor2-OST000e_UUID 49072 - 0 - - - - - pfs2wor2-OST000f_UUID 50076 - 0 - - - - - pfs2wor2-OST0010_UUID 50376 - 0 - - - - - pfs2wor2-OST0011_UUID 147536 - 0 - - - - - pfs2wor2-OST0012_UUID 35536 - 0 - - - - - pfs2wor2-OST0013_UUID 32052 - 0 - - - - - pfs2wor2-OST0014_UUID 44112 - 0 - - - - - pfs2wor2-OST0015_UUID 181624 - 0 - - - - - pfs2wor2-OST0016_UUID 41412 - 0 - - - - - pfs2wor2-OST0017_UUID 34656 - 0 - - - - - pfs2wor2-OST0018_UUID 0 - 0 - - - - - pfs2wor2-OST0019_UUID 31452 - 0 - - - - - pfs2wor2-OST001a_UUID 59996 - 0 - - - - - pfs2wor2-OST001b_UUID 148 - 0 - - - - - pfs2wor2-OST001c_UUID 38304 - 0 - - - - - pfs2wor2-OST001d_UUID 35976 - 0 - - - - - pfs2wor2-OST001e_UUID 36404 - 0 - - - - - pfs2wor2-OST001f_UUID 12 - 0 - - - - - pfs2wor2-OST0020_UUID 46252 - 0 - - - - - pfs2wor2-OST0021_UUID 38584 - 0 - - - - - pfs2wor2-OST0022_UUID 41472 - 0 - - - - - pfs2wor2-OST0023_UUID 182340 - 0 - - - - - pfs2wor2-OST0024_UUID 40308 - 0 - - - - - pfs2wor2-OST0025_UUID 33164 - 0 - - - - - pfs2wor2-OST0026_UUID 76312 - 0 - - - - - pfs2wor2-OST0027_UUID 37684 - 0 - - - - - root@uc1n990:/pfs/work2# lfs find es/es_es/es_asaramet --obd pfs2wor2-OST0007_UUID es/es_es/es_asaramet/openfoam/ThirdParty-2.2.2/openmpi-1.6.3/contrib/platform/cray_xt3 es/es_es/es_asaramet/openfoam/ThirdParty-2.2.2/openmpi-1.6.3/contrib/platform/ornl/ornl_configure.gnu es/es_es/es_asaramet/openfoam/ThirdParty-2.2.2/openmpi-1.6.3/contrib/platform/lanl/tlcc/optimized-panasas.conf es/es_es/es_asaramet/openfoam/ThirdParty-2.2.2/openmpi-1.6.3/test/datatype/opal_ddt_lib.c es/es_es/es_asaramet/openfoam/ThirdParty-2.2.2/openmpi-1.6.3/orte/runtime/orte_mca_params.c ... root@uc1n990:/pfs/work2# cat /proc/fs/lustre/lov/pfs2wor2*/target_obd 0: pfs2wor2-OST0000_UUID ACTIVE 1: pfs2wor2-OST0001_UUID ACTIVE 2: pfs2wor2-OST0002_UUID ACTIVE 3: pfs2wor2-OST0003_UUID ACTIVE 4: pfs2wor2-OST0004_UUID ACTIVE 5: pfs2wor2-OST0005_UUID ACTIVE 6: pfs2wor2-OST0006_UUID ACTIVE 7: pfs2wor2-OST0007_UUID ACTIVE 8: pfs2wor2-OST0008_UUID ACTIVE ... root@uc1n990:/pfs/work2# lfs getstripe es/es_es/es_asaramet/openfoam/OpenFOAM-2.2.2/src/mesh/autoMesh/Make/linux64GccDPOpt/refinementSurfaces.o es/es_es/es_asaramet/openfoam/OpenFOAM-2.2.2/src/mesh/autoMesh/Make/linux64GccDPOpt/refinementSurfaces.o lmm_stripe_count: 1 lmm_stripe_size: 1048576 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 988599 0xf15b7 0 root@uc1n990:/pfs/work2# ls -lh es/es_es/es_asaramet/openfoam/OpenFOAM-2.2.2/src/mesh/autoMesh/Make/linux64GccDPOpt/refinementSurfaces.o -rw-r--r-- 1 es_asaramet es_es 197K 29. Nov 13:22 es/es_es/es_asaramet/openfoam/OpenFOAM-2.2.2/src/mesh/autoMesh/Make/linux64GccDPOpt/refinementSurfaces.o root@uc1n990:/pfs/work2# ~er2341/bin/check_all_quotas -u -c 'echo es_asaramet' -f . File system .: Account Used file system blocks Used OST blocks Difference (%) es_asaramet 2036560 2001384 1.8 # At 2014-02-10 checked that the output above is still the same. # We want to check if we can get more details about the underlying ldiskfs # during production according to chapter 13.14 of the Lustre manual. # The reason is the following question in LU-4504: Johann Lombardi added a comment - 31/Jan/14 9:41 PM I wonder whether some of the OST objects belonging to those users did not get the proper UID/GID. Could you please check on one of the OST reporting the wrong usage if all the objects have the correct UID/GID? You can do it by unmounting the OST, mounting it with -t ldiskfs and run a find command to compute usage of the user and compare it with what is reported by lfs quota. Thanks in advance. # We want to check object ID 988599 on OST idx 7. [root@pfs2n12 ~]# pdsh -w pfs2n[18-21] -f 1 'cd /proc/fs/lustre/osd-ldiskfs; for i in `ls -d */* |grep mntdev`; do echo $i; cat $i; done' | grep -A1 OST0007 pfs2n18: pfs2wor2-OST0007/mntdev pfs2n18: /dev/mapper/ost_pfs2wor2_7 [root@pfs2n12 ~]# ssh pfs2n18 [root@pfs2n18 ~]# debugfs -c -R "stat /O/0/d$((988599 % 32))/988599" /dev/mapper/ost_pfs2wor2_7 debugfs 1.42.7.wc2 (07-Nov-2013) /dev/mapper/ost_pfs2wor2_7: catastrophic mode - not reading inode or group bitmaps Inode: 138413113 Type: regular Mode: 0666 Flags: 0x80000 Generation: 3765895724 Version: 0x00000000:011a1926 User: 900044 Group: 500012 Size: 201712 File ACL: 0 Directory ACL: 0 Links: 1 Blockcount: 400 Fragment: Address: 0 Number: 0 Size: 0 ctime: 0x52988719:00000000 -- Fri Nov 29 13:22:49 2013 atime: 0x52988720:00000000 -- Fri Nov 29 13:22:56 2013 mtime: 0x52988719:00000000 -- Fri Nov 29 13:22:49 2013 crtime: 0x529874d0:6d5788f0 -- Fri Nov 29 12:04:48 2013 Size of extra inode fields: 28 Extended attributes stored in inode body: fid = "07 53 00 00 02 00 00 00 b0 a0 01 00 00 00 00 00 b7 15 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 " (32) fid: objid=988599 seq=0 parent=[0x200005307:0x1a0b0:0x0] stripe=0 EXTENTS: (0-49):4429229255-4429229304 root@uc1n990:/pfs/work2# ls -ln es/es_es/es_asaramet/openfoam/OpenFOAM-2.2.2/src/mesh/autoMesh/Make/linux64GccDPOpt/refinementSurfaces.o -rw-r--r-- 1 900044 500012 201712 29. Nov 13:22 es/es_es/es_asaramet/openfoam/OpenFOAM-2.2.2/src/mesh/autoMesh/Make/linux64GccDPOpt/refinementSurfaces.o # User and group ID is correct.