[LU-13491] lfs changelog command not working on 2.12.4 client to 2.10.7 server

Thu, 11 Jun 2020 16:06:27 +0000

So I have a question about running the lfs changelog command. We've registered the changelog user on the MDT and are attempting to run the lfs changelog command against a few records to test.

On MDS:

[root@holylfs02mds02 ~]# lctl get_param mdd.holylfs2-MDT0000.changelog_users
mdd.holylfs2-MDT0000.changelog_users=
current index: 16122510026
ID index
cl3 16122503691

On Client:

/usr/bin/lfs changelog holylfs2-MDT0000 16122503691 16122503699

hangs...

No Dmesg output at all on either the client or the MDS but when I strace the command on the client I get this:

strace /usr/bin/lfs changelog holylfs2-MDT0000 16122503691 16122503699
execve("/usr/bin/lfs", ["/usr/bin/lfs", "changelog", "holylfs2-MDT0000", "16122503691", "16122503699"], [/* 32 vars */]) = 0
brk(NULL) = 0xeef000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1e62000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=67378, ...}) = 0
mmap(NULL, 67378, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f7ef1e51000
close(3) = 0
open("/lib64/liblustreapi.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 s\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=159104, ...}) = 0
mmap(NULL, 2288240, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef1a13000
mprotect(0x7f7ef1a38000, 2093056, PROT_NONE) = 0
mmap(0x7f7ef1c37000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24000) = 0x7f7ef1c37000
mmap(0x7f7ef1c39000, 35440, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1c39000
close(3) = 0
open("/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=90248, ...}) = 0
mmap(NULL, 2183272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef17fd000
mprotect(0x7f7ef1812000, 2093056, PROT_NONE) = 0
mmap(0x7f7ef1a11000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f7ef1a11000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1e50000
open("/lib64/liblnetconfig.so.4", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300X\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=109600, ...}) = 0
mmap(NULL, 2235136, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef15db000
mprotect(0x7f7ef15f4000, 2093056, PROT_NONE) = 0
mmap(0x7f7ef17f3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x7f7ef17f3000
mmap(0x7f7ef17f5000, 31488, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f7ef17f5000
close(3) = 0
open("/lib64/libyaml-0.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\32\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=131096, ...}) = 0
mmap(NULL, 2224456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef13bb000
mprotect(0x7f7ef13da000, 2093056, PROT_NONE) = 0
mmap(0x7f7ef15d9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7f7ef15d9000
close(3) = 0
open("/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0PS\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1136944, ...}) = 0
mmap(NULL, 3150136, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef10b9000
mprotect(0x7f7ef11ba000, 2093056, PROT_NONE) = 0
mmap(0x7f7ef13b9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x100000) = 0x7f7ef13b9000
close(3) = 0
open("/lib64/libreadline.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0PO\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=285240, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1e4f000
mmap(NULL, 2380744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef0e73000
mprotect(0x7f7ef0eaf000, 2097152, PROT_NONE) = 0
mmap(0x7f7ef10af000, 32768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3c000) = 0x7f7ef10af000
mmap(0x7f7ef10b7000, 5064, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f7ef10b7000
close(3) = 0
open("/lib64/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=15688, ...}) = 0
mmap(NULL, 2109720, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef0c6f000
mprotect(0x7f7ef0c72000, 2093056, PROT_NONE) = 0
mmap(0x7f7ef0e71000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f7ef0e71000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`&\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2156240, ...}) = 0
mmap(NULL, 3985920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef08a1000
mprotect(0x7f7ef0a64000, 2097152, PROT_NONE) = 0
mmap(0x7f7ef0c64000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c3000) = 0x7f7ef0c64000
mmap(0x7f7ef0c6a000, 16896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f7ef0c6a000
close(3) = 0
open("/lib64/libtinfo.so.5", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\316\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=174576, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1e4e000
mmap(NULL, 2268928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef0677000
mprotect(0x7f7ef069c000, 2097152, PROT_NONE) = 0
mmap(0x7f7ef089c000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7f7ef089c000
close(3) = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19248, ...}) = 0
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f7ef0473000
mprotect(0x7f7ef0475000, 2097152, PROT_NONE) = 0
mmap(0x7f7ef0675000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f7ef0675000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1e4d000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f7ef1e4b000
arch_prctl(ARCH_SET_FS, 0x7f7ef1e4b740) = 0
mprotect(0x7f7ef0c64000, 16384, PROT_READ) = 0
mprotect(0x7f7ef0675000, 4096, PROT_READ) = 0
mprotect(0x7f7ef089c000, 16384, PROT_READ) = 0
mprotect(0x7f7ef0e71000, 4096, PROT_READ) = 0
mprotect(0x7f7ef10af000, 8192, PROT_READ) = 0
mprotect(0x7f7ef13b9000, 4096, PROT_READ) = 0
mprotect(0x7f7ef15d9000, 4096, PROT_READ) = 0
mprotect(0x7f7ef17f3000, 4096, PROT_READ) = 0
mprotect(0x7f7ef1a11000, 4096, PROT_READ) = 0
mprotect(0x7f7ef1c37000, 4096, PROT_READ) = 0
mprotect(0x625000, 4096, PROT_READ) = 0
mprotect(0x7f7ef1e63000, 4096, PROT_READ) = 0
munmap(0x7f7ef1e51000, 67378) = 0
gettid() = 32258
open("/dev/urandom", O_RDONLY|O_NOFOLLOW) = 3
read(3, "\27\344\247+", 4) = 4
close(3) = 0
brk(NULL) = 0xeef000
brk(0xf10000) = 0xf10000
brk(NULL) = 0xf10000
open("/dev/changelog-holylfs2-MDT0000", O_RDONLY) = 3
lseek(3, 16122503691, SEEK_SET) = 16122503691
read(3

Any clue as to why it's hanging trying to read /dev/changelog-holylfs2-MDT0000. The device does seem to exist when I run this. This connection is over IB and is not running over lnet routes. I'm running 2.12.4 on the client and the server is 2.10.7. Any help would be much appreciated.

Whamcloud Community JIRA

[LU-13491] lfs changelog command not working on 2.12.4 client to 2.10.7 server