# This file describes further investigations according to # SR 28763 and LU-4504. root@uc1n990:/pfs/work2# lfs quota -v -u es_asaramet . Disk quotas for user es_asaramet (uid 900044): Filesystem kbytes quota limit grace files quota limit grace . 1608776 0 0 - 78 0 0 - pfs2wor2-MDT0000_UUID 88 - 0 - 78 - 0 - pfs2wor2-OST0000_UUID 1000 - 0 - - - - - pfs2wor2-OST0001_UUID 340164 - 0 - - - - - pfs2wor2-OST0002_UUID 221188 - 0 - - - - - pfs2wor2-OST0003_UUID 0 - 0 - - - - - pfs2wor2-OST0004_UUID 12 - 0 - - - - - pfs2wor2-OST0005_UUID 4 - 0 - - - - - pfs2wor2-OST0006_UUID 0 - 0 - - - - - pfs2wor2-OST0007_UUID 328 - 0 - - - - - pfs2wor2-OST0008_UUID 12 - 0 - - - - - pfs2wor2-OST0009_UUID 50180 - 0 - - - - - pfs2wor2-OST000a_UUID 276 - 0 - - - - - pfs2wor2-OST000b_UUID 0 - 0 - - - - - pfs2wor2-OST000c_UUID 221220 - 0 - - - - - pfs2wor2-OST000d_UUID 8 - 0 - - - - - pfs2wor2-OST000e_UUID 12 - 0 - - - - - pfs2wor2-OST000f_UUID 0 - 0 - - - - - pfs2wor2-OST0010_UUID 8 - 0 - - - - - pfs2wor2-OST0011_UUID 109092 - 0 - - - - - pfs2wor2-OST0012_UUID 4 - 0 - - - - - pfs2wor2-OST0013_UUID 49236 - 0 - - - - - pfs2wor2-OST0014_UUID 47112 - 0 - - - - - pfs2wor2-OST0015_UUID 76 - 0 - - - - - pfs2wor2-OST0016_UUID 86668 - 0 - - - - - pfs2wor2-OST0017_UUID 4 - 0 - - - - - pfs2wor2-OST0018_UUID 8 - 0 - - - - - pfs2wor2-OST0019_UUID 112 - 0 - - - - - pfs2wor2-OST001a_UUID 12 - 0 - - - - - pfs2wor2-OST001b_UUID 347104 - 0 - - - - - pfs2wor2-OST001c_UUID 60 - 0 - - - - - pfs2wor2-OST001d_UUID 8 - 0 - - - - - pfs2wor2-OST001e_UUID 0 - 0 - - - - - pfs2wor2-OST001f_UUID 8 - 0 - - - - - pfs2wor2-OST0020_UUID 0 - 0 - - - - - pfs2wor2-OST0021_UUID 87048 - 0 - - - - - pfs2wor2-OST0022_UUID 47708 - 0 - - - - - pfs2wor2-OST0023_UUID 4 - 0 - - - - - pfs2wor2-OST0024_UUID 8 - 0 - - - - - pfs2wor2-OST0025_UUID 4 - 0 - - - - - pfs2wor2-OST0026_UUID 0 - 0 - - - - - pfs2wor2-OST0027_UUID 0 - 0 - - - - - root@uc1n990:/pfs/work2# lfs find es/es_es/es_asaramet --obd pfs2wor2-OST0027_UUID | xargs ls -l -rw-r--r-- 1 es_asaramet es_es 70749 10. Mär 09:28 es/es_es/es_asaramet/mesh/meshScript/constant/triSurface/tauchrohre.stl -rwxr-xr-x 1 es_asaramet es_es 1962 10. Feb 15:32 es/es_es/es_asaramet/simple/UPsimple/system/fvSchemes root@uc1n990:/pfs/work2# cat /proc/fs/lustre/lov/pfs2wor2*/target_obd | grep OST0027 39: pfs2wor2-OST0027_UUID ACTIVE root@uc1n990:/pfs/work2# lfs getstripe es/es_es/es_asaramet/mesh/meshScript/constant/triSurface/tauchrohre.stl es/es_es/es_asaramet/mesh/meshScript/constant/triSurface/tauchrohre.stl lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_layout_gen: 0 lmm_stripe_offset: 17 obdidx objid objid group 17 10295731 0x9d19b3 0 39 7178541 0x6d892d 0 root@uc1n990:/pfs/work2# lfs getstripe es/es_es/es_asaramet/simple/UPsimple/system/fvSchemes es/es_es/es_asaramet/simple/UPsimple/system/fvSchemes lmm_stripe_count: 2 lmm_stripe_size: 1048576 lmm_layout_gen: 0 lmm_stripe_offset: 17 obdidx objid objid group 17 10388080 0x9e8270 0 39 7269931 0x6eee2b 0 # There is indeed no data on OST0027. root@uc1n990:/pfs/work2# ~er2341/bin/check_all_quotas -u -c 'echo es_asaramet' -f . File system .: Account Used file system blocks Used OST blocks Difference (%) es_asaramet 4329272 4329100 0.0 root@uc1n990:/pfs/work2# find es/es_es/es_asaramet ! \( -user es_asaramet \) -exec ls -l {} \; root@uc1n990:/pfs/work2# du -hs es/es_es/es_asaramet 5,4G es/es_es/es_asaramet root@uc1n990:/pfs/work2# lfs quota -u es_asaramet . Disk quotas for user es_asaramet (uid 900044): Filesystem kbytes quota limit grace files quota limit grace . 6402708 0 0 - 2257 0 0 - # At least for user es_asaramet (UID 900044) it seems that the problem was fixed. # Check for user cb0001 who has been affected on the same file system before: root@uc1n990:/pfs/work2# find kit/aoc/cb0001 ! \( -user cb0001 \) -exec ls -l {} \; root@uc1n990:/pfs/work2# du -hs kit/aoc/cb0001 4,0G kit/aoc/cb0001 root@uc1n990:/pfs/work2# lfs quota -u cb0001 . Disk quotas for user cb0001 (uid 15214): Filesystem kbytes quota limit grace files quota limit grace . 4161232 0 0 - 391 0 0 - # User cb0001 is also no longer affected. # Result: Recreating quotas on a single OST (OST0007) seems to have fixed # the problem for the pfs2wor2 file system. # Now we want to check other affected users on other file systems: root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# find . ! \( -user aj9102 \) -exec ls -l {} \; root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# du -hs . 18T . root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# lfs quota -u aj9102 . Disk quotas for user aj9102 (uid 3522): Filesystem kbytes quota limit grace files quota limit grace . 917552300 0 0 - 4215 0 0 - # User aj9102 is still affected for the pfs2wor1 file system. root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# ~er2341/bin/check_all_quotas -u -c 'echo aj9102' -f . File system .: Account Used file system blocks Used OST blocks Difference (%) aj9102 917552300 917551828 0.0 root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# lfs quota -v -u aj9102 . Disk quotas for user aj9102 (uid 3522): Filesystem kbytes quota limit grace files quota limit grace . 917552300 0 0 - 4215 0 0 - pfs2wor1-MDT0000_UUID 472 - 0 - 4215 - 0 - pfs2wor1-OST0000_UUID 38413760 - 0 - - - - - pfs2wor1-OST0001_UUID 12439348 - 0 - - - - - pfs2wor1-OST0002_UUID 23798544 - 0 - - - - - pfs2wor1-OST0003_UUID 24255468 - 0 - - - - - pfs2wor1-OST0004_UUID 40490920 - 0 - - - - - pfs2wor1-OST0005_UUID 30589204 - 0 - - - - - pfs2wor1-OST0006_UUID 23798044 - 0 - - - - - pfs2wor1-OST0007_UUID 30176696 - 0 - - - - - pfs2wor1-OST0008_UUID 6836376 - 0 - - - - - pfs2wor1-OST0009_UUID 24613016 - 0 - - - - - pfs2wor1-OST000a_UUID 40544544 - 0 - - - - - pfs2wor1-OST000b_UUID 23391768 - 0 - - - - - pfs2wor1-OST000c_UUID 24837048 - 0 - - - - - pfs2wor1-OST000d_UUID 21669404 - 0 - - - - - pfs2wor1-OST000e_UUID 16691888 - 0 - - - - - pfs2wor1-OST000f_UUID 17467872 - 0 - - - - - pfs2wor1-OST0010_UUID 16330028 - 0 - - - - - pfs2wor1-OST0011_UUID 24526136 - 0 - - - - - pfs2wor1-OST0012_UUID 24841780 - 0 - - - - - pfs2wor1-OST0013_UUID 29451168 - 0 - - - - - pfs2wor1-OST0014_UUID 12899736 - 0 - - - - - pfs2wor1-OST0015_UUID 31311580 - 0 - - - - - pfs2wor1-OST0016_UUID 4398960 - 0 - - - - - pfs2wor1-OST0017_UUID 35201252 - 0 - - - - - pfs2wor1-OST0018_UUID 12174296 - 0 - - - - - pfs2wor1-OST0019_UUID 30945216 - 0 - - - - - pfs2wor1-OST001a_UUID 17467784 - 0 - - - - - pfs2wor1-OST001b_UUID 8096940 - 0 - - - - - pfs2wor1-OST001c_UUID 18459688 - 0 - - - - - pfs2wor1-OST001d_UUID 30590608 - 0 - - - - - pfs2wor1-OST001e_UUID 12902220 - 0 - - - - - pfs2wor1-OST001f_UUID 19687880 - 0 - - - - - pfs2wor1-OST0020_UUID 27421276 - 0 - - - - - pfs2wor1-OST0021_UUID 45155280 - 0 - - - - - pfs2wor1-OST0022_UUID 30614244 - 0 - - - - - pfs2wor1-OST0023_UUID 15074664 - 0 - - - - - pfs2wor1-OST0024_UUID 15362544 - 0 - - - - - pfs2wor1-OST0025_UUID 10552416 - 0 - - - - - pfs2wor1-OST0026_UUID 28053844 - 0 - - - - - pfs2wor1-OST0027_UUID 16018388 - 0 - - - - - root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# find . -size +10G -exec ls -l {} \; root@ic2n993:/pfs/imk/imk-tro/ACP/aj9102# find . -size +9G -exec ls -l {} \; | wc 1421 12789 168285 # Result: User aj9102 is still affected. This user has many large files and # we have no idea to find out which OST might be affected. # Check another user on file system pfs2dat2: root@ic2n993:/pfs/data2/home# find kn/kn_kn/kn_pop164377 ! \( -user kn_pop164377 \) -exec ls -l {} \; root@ic2n993:/pfs/data2/home# du -hs kn/kn_kn/kn_pop164377 165G kn/kn_kn/kn_pop164377 root@ic2n993:/pfs/data2/home# lfs quota -u kn_pop164377 . Disk quotas for user kn_pop164377 (uid 900025): Filesystem kbytes quota limit grace files quota limit grace . 48485720 0 0 - 380559 0 0 - root@ic2n993:/pfs/data2/home# lfs quota -v -u kn_pop164377 . Disk quotas for user kn_pop164377 (uid 900025): Filesystem kbytes quota limit grace files quota limit grace . 48543584 0 0 - 380591 0 0 - pfs2dat2-MDT0000_UUID 49116 - 0 - 380591 - 0 - pfs2dat2-OST0000_UUID 2420956 - 0 - - - - - pfs2dat2-OST0001_UUID 2527344 - 0 - - - - - pfs2dat2-OST0002_UUID 2436964 - 0 - - - - - pfs2dat2-OST0003_UUID 2355504 - 0 - - - - - pfs2dat2-OST0004_UUID 2413956 - 0 - - - - - pfs2dat2-OST0005_UUID 2261340 - 0 - - - - - pfs2dat2-OST0006_UUID 2579196 - 0 - - - - - pfs2dat2-OST0007_UUID 2360768 - 0 - - - - - pfs2dat2-OST0008_UUID 2577796 - 0 - - - - - pfs2dat2-OST0009_UUID 2437720 - 0 - - - - - pfs2dat2-OST000a_UUID 2364652 - 0 - - - - - pfs2dat2-OST000b_UUID 2411436 - 0 - - - - - pfs2dat2-OST000c_UUID 2396652 - 0 - - - - - pfs2dat2-OST000d_UUID 2357692 - 0 - - - - - pfs2dat2-OST000e_UUID 2442208 - 0 - - - - - pfs2dat2-OST000f_UUID 2403904 - 0 - - - - - pfs2dat2-OST0010_UUID 2368284 - 0 - - - - - pfs2dat2-OST0011_UUID 2486908 - 0 - - - - - pfs2dat2-OST0012_UUID 2395624 - 0 - - - - - pfs2dat2-OST0013_UUID 2495564 - 0 - - - - - root@ic2n993:/pfs/data2/home# find kn/kn_kn/kn_pop164377 -size +3G -exec ls -l {} \; # Result: User kn_pop164377 is still affected. This user has many files and # we have no idea to find out which OST might be affected. # Check another user on file system pfs2dat1: root@uc1n990:/pfs/data1/home/kit# find imk-tro/ks2420 ! \( -user ks2420 \) -exec ls -l {} \; -rw-r--r-- 1 22490 4512 447511194 24. Jun 2012 imk-tro/ks2420/Emissionen/VEU_test/EMfile root@uc1n990:/pfs/data1/home/kit# du -hs imk-tro/ks2420 3,8T imk-tro/ks2420 root@uc1n990:/pfs/data1/home/kit# lfs quota -u ks2420 . Disk quotas for user ks2420 (uid 3512): Filesystem kbytes quota limit grace files quota limit grace . 9352704 0 0 - 152846 0 0 - root@uc1n990:/pfs/data1/home/kit# lfs quota -v -u ks2420 . Disk quotas for user ks2420 (uid 3512): Filesystem kbytes quota limit grace files quota limit grace . 9352704 0 0 - 152846 0 0 - pfs2dat1-MDT0000_UUID 43760 - 0 - 152846 - 0 - pfs2dat1-OST0000_UUID 819352 - 0 - - - - - pfs2dat1-OST0001_UUID 148576 - 0 - - - - - pfs2dat1-OST0002_UUID 193916 - 0 - - - - - pfs2dat1-OST0003_UUID 23652 - 0 - - - - - pfs2dat1-OST0004_UUID 263276 - 0 - - - - - pfs2dat1-OST0005_UUID 920228 - 0 - - - - - pfs2dat1-OST0006_UUID 396568 - 0 - - - - - pfs2dat1-OST0007_UUID 169840 - 0 - - - - - pfs2dat1-OST0008_UUID 254616 - 0 - - - - - pfs2dat1-OST0009_UUID 218120 - 0 - - - - - pfs2dat1-OST000a_UUID 1219732 - 0 - - - - - pfs2dat1-OST000b_UUID 111536 - 0 - - - - - pfs2dat1-OST000c_UUID 2142044 - 0 - - - - - pfs2dat1-OST000d_UUID 668428 - 0 - - - - - pfs2dat1-OST000e_UUID 152820 - 0 - - - - - pfs2dat1-OST000f_UUID 135224 - 0 - - - - - pfs2dat1-OST0010_UUID 222620 - 0 - - - - - pfs2dat1-OST0011_UUID 94376 - 0 - - - - - pfs2dat1-OST0012_UUID 101624 - 0 - - - - - pfs2dat1-OST0013_UUID 1052396 - 0 - - - - - root@uc1n990:/pfs/data1/home/kit# find imk-tro/ks2420 -size +10G -exec ls -l {} \; -rw-r--r-- 1 ks2420 imk-tro-acp 27276309704 4. Feb 2013 imk-tro/ks2420/lundgren/DATA/Copy_from_AIX/INPUT/Input_Geo/GME/FILES_gffxx_grb.tar.Z root@uc1n990:/pfs/data1/home/kit# lfs getstripe imk-tro/ks2420/lundgren/DATA/Copy_from_AIX/INPUT/Input_Geo/GME/FILES_gffxx_grb.tar.Z imk-tro/ks2420/lundgren/DATA/Copy_from_AIX/INPUT/Input_Geo/GME/FILES_gffxx_grb.tar.Z lmm_stripe_count: 1 lmm_stripe_size: 1048576 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 8902758 0x87d866 0 root@uc1n990:/pfs/data1/home/kit# cat /proc/fs/lustre/lov/pfs2dat1*/target_obd | egrep "^2:" 2: pfs2dat1-OST0002_UUID ACTIVE # Result: User ks2420 is still affected. This user has one 26GB file # on OST0002 but lfs quota on shows only 193916 KB. Hence # quotas on OST0002 are possibly bad.