[LU-15075] Moving file fails with "No data available" error

• Status: Open
• Project: Lustre
• Affects Version/s: Lustre 2.12.6
• Type: Bug
• Priority: Minor
• Reporter: Steven Vandenbrande
• Assignee: WC Triage
• Resolution: Unresolved

Environment

CentOS Linux release 7.9.2009

• Severity: 3

Description

Several users of our Lustre file system report that once in a while they generate a file which cannot be moved. It does not happen very frequently (maybe once every several thousand files), but it is enough to disrupt the workflow of their calculations.

The following if for an example text file of 5461 bytes, which looks normal when running cat on it. However, moving the file is impossible:

# mv _jobinfo.tmp _jobinfo2
mv: cannot move ‘_jobinfo.tmp’ to ‘_jobinfo2’: No data available

Weirdly, by changing the target file name longer, all of a sudden it works:

# mv _jobinfo.tmp _jobinfo3 # This works!

 Here is the strace of both commands:

# strace mv _jobinfo.tmp _jobinfo2
execve("/usr/bin/mv", ["mv", "_jobinfo.tmp", "_jobinfo2"], 0x7ffdcfdc1a80 /* 67 vars */) = 0
brk(NULL)                               = 0xfdb000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f998ad000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=59519, ...}) = 0
mmap(NULL, 59519, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9f9989e000
close(3)                                = 0
open("/usr/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220j\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=155744, ...}) = 0
mmap(NULL, 2255216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f99466000
mprotect(0x7f9f9948a000, 2093056, PROT_NONE) = 0
mmap(0x7f9f99689000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f9f99689000
mmap(0x7f9f9968b000, 6512, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f9f9968b000
close(3)                                = 0
open("/usr/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\37\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=37064, ...}) = 0
mmap(NULL, 2130560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f9925d000
mprotect(0x7f9f99264000, 2097152, PROT_NONE) = 0
mmap(0x7f9f99464000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f9f99464000
close(3)                                = 0
open("/usr/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19896, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f9989d000
mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f99058000
mprotect(0x7f9f9905c000, 2093056, PROT_NONE) = 0
mmap(0x7f9f9925b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f9f9925b000
close(3)                                = 0
open("/usr/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`&\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2156344, ...}) = 0
mmap(NULL, 3985920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f98c8a000
mprotect(0x7f9f98e4e000, 2093056, PROT_NONE) = 0
mmap(0x7f9f9904d000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c3000) = 0x7f9f9904d000
mmap(0x7f9f99053000, 16896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f9f99053000
close(3)                                = 0
open("/usr/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=402384, ...}) = 0
mmap(NULL, 2494984, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f98a28000
mprotect(0x7f9f98a88000, 2097152, PROT_NONE) = 0
mmap(0x7f9f98c88000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x60000) = 0x7f9f98c88000
close(3)                                = 0
open("/usr/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19248, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f9989c000
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f98824000
mprotect(0x7f9f98826000, 2097152, PROT_NONE) = 0
mmap(0x7f9f98a26000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f9f98a26000
close(3)                                = 0
open("/usr/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200m\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=142144, ...}) = 0
mmap(NULL, 2208904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f98608000
mprotect(0x7f9f9861f000, 2093056, PROT_NONE) = 0
mmap(0x7f9f9881e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f9f9881e000
mmap(0x7f9f98820000, 13448, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f9f98820000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f9989b000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f99899000
arch_prctl(ARCH_SET_FS, 0x7f9f99899840) = 0
mprotect(0x7f9f9904d000, 16384, PROT_READ) = 0
mprotect(0x7f9f9881e000, 4096, PROT_READ) = 0
mprotect(0x7f9f98a26000, 4096, PROT_READ) = 0
mprotect(0x7f9f98c88000, 4096, PROT_READ) = 0
mprotect(0x7f9f9925b000, 4096, PROT_READ) = 0
mprotect(0x7f9f99464000, 4096, PROT_READ) = 0
mprotect(0x7f9f99689000, 4096, PROT_READ) = 0
mprotect(0x61d000, 4096, PROT_READ)     = 0
mprotect(0x7f9f998ae000, 4096, PROT_READ) = 0
munmap(0x7f9f9989e000, 59519)           = 0
set_tid_address(0x7f9f99899b10)         = 23764
set_robust_list(0x7f9f99899b20, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f9f9860e860, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f9f98617630}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f9f9860e8f0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f9f98617630}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=300000*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffc081f95b0) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffc081f95b0)      = -1 ENOENT (No such file or directory)
brk(NULL)                               = 0xfdb000
brk(0xffc000)                           = 0xffc000
open("/proc/filesystems", O_RDONLY)     = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f998ac000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 471
stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffc081f8af0) = -1 ENOENT (No such file or directory)
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7f9f998ac000, 4096)            = 0
access("/etc/selinux/config", F_OK)     = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106172832, ...}) = 0
mmap(NULL, 106172832, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9f920c6000
close(3)                                = 0
geteuid()                               = 0
ioctl(0, TCGETS, {B38400 opost isig icanon echo ...}) = 0
stat("_jobinfo2", 0x7ffc081f9560)       = -1 ENOENT (No such file or directory)
lstat("_jobinfo.tmp", {st_mode=S_IFREG|0644, st_size=5461, ...}) = 0
lstat("_jobinfo2", 0x7ffc081f9210)      = -1 ENOENT (No such file or directory)
renameat2(AT_FDCWD, "_jobinfo.tmp", AT_FDCWD, "_jobinfo2", 0) = -1 ENODATA (No data available)
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f998ac000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7f9f998ac000, 4096)            = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib64/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mv: ", 4mv: )                     = 4
write(2, "cannot move \342\200\230_jobinfo.tmp\342\200\231 t"..., 49cannot move ‘_jobinfo.tmp’ to ‘_jobinfo2’) = 49
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, ": No data available", 19: No data available)     = 19
write(2, "\n", 1
)                       = 1
lseek(0, 0, SEEK_CUR)                   = -1 ESPIPE (Illegal seek)
close(0)                                = 0
close(1)                                = 0
close(2)                                = 0
exit_group(1)                           = ?
+++ exited with 1 +++

 
# strace mv _jobinfo.tmp _jobinfo3 execve("/usr/bin/mv", ["mv", "_jobinfo.tmp", "_jobinfo3"], 0x7ffd4e5b37a0 /* 67 vars */) = 0 brk(NULL) = 0x871000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e5248d000 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=59519, ...}) = 0 mmap(NULL, 59519, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3e5247e000 close(3) = 0 open("/usr/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220j\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=155744, ...}) = 0 mmap(NULL, 2255216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e52046000 mprotect(0x7f3e5206a000, 2093056, PROT_NONE) = 0 mmap(0x7f3e52269000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f3e52269000 mmap(0x7f3e5226b000, 6512, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3e5226b000 close(3) = 0 open("/usr/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\37\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=37064, ...}) = 0 mmap(NULL, 2130560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e51e3d000 mprotect(0x7f3e51e44000, 2097152, PROT_NONE) = 0 mmap(0x7f3e52044000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f3e52044000 close(3) = 0 open("/usr/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=19896, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e5247d000 mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e51c38000 mprotect(0x7f3e51c3c000, 2093056, PROT_NONE) = 0 mmap(0x7f3e51e3b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f3e51e3b000 close(3) = 0 open("/usr/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`&\2\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=2156344, ...}) = 0 mmap(NULL, 3985920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e5186a000 mprotect(0x7f3e51a2e000, 2093056, PROT_NONE) = 0 mmap(0x7f3e51c2d000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c3000) = 0x7f3e51c2d000 mmap(0x7f3e51c33000, 16896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3e51c33000 close(3) = 0 open("/usr/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=402384, ...}) = 0 mmap(NULL, 2494984, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e51608000 mprotect(0x7f3e51668000, 2097152, PROT_NONE) = 0 mmap(0x7f3e51868000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x60000) = 0x7f3e51868000 close(3) = 0 open("/usr/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\16\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=19248, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e5247c000 mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e51404000 mprotect(0x7f3e51406000, 2097152, PROT_NONE) = 0 mmap(0x7f3e51606000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f3e51606000 close(3) = 0 open("/usr/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200m\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=142144, ...}) = 0 mmap(NULL, 2208904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3e511e8000 mprotect(0x7f3e511ff000, 2093056, PROT_NONE) = 0 mmap(0x7f3e513fe000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3e513fe000 mmap(0x7f3e51400000, 13448, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3e51400000 close(3) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e5247b000 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e52479000 arch_prctl(ARCH_SET_FS, 0x7f3e52479840) = 0 mprotect(0x7f3e51c2d000, 16384, PROT_READ) = 0 mprotect(0x7f3e513fe000, 4096, PROT_READ) = 0 mprotect(0x7f3e51606000, 4096, PROT_READ) = 0 mprotect(0x7f3e51868000, 4096, PROT_READ) = 0 mprotect(0x7f3e51e3b000, 4096, PROT_READ) = 0 mprotect(0x7f3e52044000, 4096, PROT_READ) = 0 mprotect(0x7f3e52269000, 4096, PROT_READ) = 0 mprotect(0x61d000, 4096, PROT_READ) = 0 mprotect(0x7f3e5248e000, 4096, PROT_READ) = 0 munmap(0x7f3e5247e000, 59519) = 0 set_tid_address(0x7f3e52479b10) = 22851 set_robust_list(0x7f3e52479b20, 24) = 0 rt_sigaction(SIGRTMIN, {sa_handler=0x7f3e511ee860, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f3e511f7630}, NULL, 8) = 0 rt_sigaction(SIGRT_1, {sa_handler=0x7f3e511ee8f0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f3e511f7630}, NULL, 8) = 0 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 getrlimit(RLIMIT_STACK, {rlim_cur=300000*1024, rlim_max=RLIM64_INFINITY}) = 0 statfs("/sys/fs/selinux", 0x7ffc229f7b50) = -1 ENOENT (No such file or directory) statfs("/selinux", 0x7ffc229f7b50) = -1 ENOENT (No such file or directory) brk(NULL) = 0x871000 brk(0x892000) = 0x892000 open("/proc/filesystems", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e5248c000 read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 471 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffc229f70b0) = -1 ENOENT (No such file or directory) read(3, "", 1024) = 0 close(3) = 0 munmap(0x7f3e5248c000, 4096) = 0 access("/etc/selinux/config", F_OK) = 0 open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=106172832, ...}) = 0 mmap(NULL, 106172832, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3e4aca6000 close(3) = 0 geteuid() = 0 ioctl(0, TCGETS, {B38400 opost isig icanon echo ...}) = 0 stat("_jobinfo3", 0x7ffc229f7b00) = -1 ENOENT (No such file or directory) lstat("_jobinfo.tmp", {st_mode=S_IFREG|0644, st_size=5461, ...}) = 0 lstat("_jobinfo3", 0x7ffc229f77b0) = -1 ENOENT (No such file or directory) renameat2(AT_FDCWD, "_jobinfo.tmp", AT_FDCWD, "_jobinfo3", 0) = 0 lseek(0, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) close(0) = 0 close(1) = 0 close(2) = 0 exit_group(0) = ? +++ exited with 0 +++

The issue only occurs on our Lustre file system. It is currently not clear how to consistently generate a file for which the problem is there. The precise lustre client version is lustre-client-2.12.6-1k3.10.0v1160.11.1.el7ib4.9.x86_64
There are some other tickets with a similar error, but I could not find a solution that could be relevant for this problem (https://www.mail-archive.com/[e-mail]/msg16584.html - https://jira.whamcloud.com/browse/LU-8039)

Comments

[Taizeng Wu]

I have same issue just like https://www.mail-archive.com/[e-mail]/msg16584.html without upgrade lustre, and the mds server report:

Nov 8 04:17:57 mds1 kernel: Lustre: 17939:0: (osd handler.c:501:osd ldiskfs add entr
V()) Skipped 156 previous similar messages
Nov 8 04:17:57 mds1 kernel: LustreError: 17939:0: (osd handler.c:4570: osd xattr hand
le linkea()) share-MDT0000: failed to create agent entry for [0x200000dab: 0xb801: 0x0]: rc = -28
 

 

There are four MDTs in our environment, and I now temporarily mask the problem by limiting the directory to one MDT by `lfs setdirstripe -c 1 ...`

• OS: CentOS 7.8
• Lustre: 2.12.4