[ 0.000000] Initializing cgroup subsys cpuset [ 0.000000] Initializing cgroup subsys cpu [ 0.000000] Initializing cgroup subsys cpuacct [ 0.000000] Linux version 3.10.0-957.27.2.el7_lustre.pl2.x86_64 (sthiell@oak-rbh01) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-39) (GCC) ) #1 SMP Thu Nov 7 15:26:16 PST 2019 [ 0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-3.10.0-957.27.2.el7_lustre.pl2.x86_64 root=UUID=13d7db90-76a6-4160-92a3-3d6e156edf61 ro crashkernel=auto nomodeset console=ttyS0,115200 LANG=en_US.UTF-8 [ 0.000000] e820: BIOS-provided physical RAM map: [ 0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000008efff] usable [ 0.000000] BIOS-e820: [mem 0x000000000008f000-0x000000000008ffff] ACPI NVS [ 0.000000] BIOS-e820: [mem 0x0000000000090000-0x000000000009ffff] usable [ 0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000004f722fff] usable [ 0.000000] BIOS-e820: [mem 0x000000004f723000-0x000000005772bfff] reserved [ 0.000000] BIOS-e820: [mem 0x000000005772c000-0x000000006cacefff] usable [ 0.000000] BIOS-e820: [mem 0x000000006cacf000-0x000000006efcefff] reserved [ 0.000000] BIOS-e820: [mem 0x000000006efcf000-0x000000006fdfefff] ACPI NVS [ 0.000000] BIOS-e820: [mem 0x000000006fdff000-0x000000006fffefff] ACPI data [ 0.000000] BIOS-e820: [mem 0x000000006ffff000-0x000000006fffffff] usable [ 0.000000] BIOS-e820: [mem 0x0000000070000000-0x000000008fffffff] reserved [ 0.000000] BIOS-e820: [mem 0x00000000fec10000-0x00000000fec10fff] reserved [ 0.000000] BIOS-e820: [mem 0x00000000fed80000-0x00000000fed80fff] reserved [ 0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000107f37ffff] usable [ 0.000000] BIOS-e820: [mem 0x000000107f380000-0x000000107fffffff] reserved [ 0.000000] BIOS-e820: [mem 0x0000001080000000-0x000000207ff7ffff] usable [ 0.000000] BIOS-e820: [mem 0x000000207ff80000-0x000000207fffffff] reserved [ 0.000000] BIOS-e820: [mem 0x0000002080000000-0x000000307ff7ffff] usable [ 0.000000] BIOS-e820: [mem 0x000000307ff80000-0x000000307fffffff] reserved [ 0.000000] BIOS-e820: [mem 0x0000003080000000-0x000000407ff7ffff] usable [ 0.000000] BIOS-e820: [mem 0x000000407ff80000-0x000000407fffffff] reserved [ 0.000000] NX (Execute Disable) protection: active [ 0.000000] e820: update [mem 0x378df020-0x37980c5f] usable ==> usable [ 0.000000] e820: update [mem 0x378ad020-0x378dee5f] usable ==> usable [ 0.000000] e820: update [mem 0x3787b020-0x378ace5f] usable ==> usable [ 0.000000] e820: update [mem 0x37872020-0x3787a05f] usable ==> usable [ 0.000000] e820: update [mem 0x3784c020-0x37871c5f] usable ==> usable [ 0.000000] e820: update [mem 0x37833020-0x3784b65f] usable ==> usable [ 0.000000] extended physical RAM map: [ 0.000000] reserve setup_data: [mem 0x0000000000000000-0x000000000008efff] usable [ 0.000000] reserve setup_data: [mem 0x000000000008f000-0x000000000008ffff] ACPI NVS [ 0.000000] reserve setup_data: [mem 0x0000000000090000-0x000000000009ffff] usable [ 0.000000] reserve setup_data: [mem 0x0000000000100000-0x000000003783301f] usable [ 0.000000] reserve setup_data: [mem 0x0000000037833020-0x000000003784b65f] usable [ 0.000000] reserve setup_data: [mem 0x000000003784b660-0x000000003784c01f] usable [ 0.000000] reserve setup_data: [mem 0x000000003784c020-0x0000000037871c5f] usable [ 0.000000] reserve setup_data: [mem 0x0000000037871c60-0x000000003787201f] usable [ 0.000000] reserve setup_data: [mem 0x0000000037872020-0x000000003787a05f] usable [ 0.000000] reserve setup_data: [mem 0x000000003787a060-0x000000003787b01f] usable [ 0.000000] reserve setup_data: [mem 0x000000003787b020-0x00000000378ace5f] usable [ 0.000000] reserve setup_data: [mem 0x00000000378ace60-0x00000000378ad01f] usable [ 0.000000] reserve setup_data: [mem 0x00000000378ad020-0x00000000378dee5f] usable [ 0.000000] reserve setup_data: [mem 0x00000000378dee60-0x00000000378df01f] usable [ 0.000000] reserve setup_data: [mem 0x00000000378df020-0x0000000037980c5f] usable [ 0.000000] reserve setup_data: [mem 0x0000000037980c60-0x000000004f722fff] usable [ 0.000000] reserve setup_data: [mem 0x000000004f723000-0x000000005772bfff] reserved [ 0.000000] reserve setup_data: [mem 0x000000005772c000-0x000000006cacefff] usable [ 0.000000] reserve setup_data: [mem 0x000000006cacf000-0x000000006efcefff] reserved [ 0.000000] reserve setup_data: [mem 0x000000006efcf000-0x000000006fdfefff] ACPI NVS [ 0.000000] reserve setup_data: [mem 0x000000006fdff000-0x000000006fffefff] ACPI data [ 0.000000] reserve setup_data: [mem 0x000000006ffff000-0x000000006fffffff] usable [ 0.000000] reserve setup_data: [mem 0x0000000070000000-0x000000008fffffff] reserved [ 0.000000] reserve setup_data: [mem 0x00000000fec10000-0x00000000fec10fff] reserved [ 0.000000] reserve setup_data: [mem 0x00000000fed80000-0x00000000fed80fff] reserved [ 0.000000] reserve setup_data: [mem 0x0000000100000000-0x000000107f37ffff] usable [ 0.000000] reserve setup_data: [mem 0x000000107f380000-0x000000107fffffff] reserved [ 0.000000] reserve setup_data: [mem 0x0000001080000000-0x000000207ff7ffff] usable [ 0.000000] reserve setup_data: [mem 0x000000207ff80000-0x000000207fffffff] reserved [ 0.000000] reserve setup_data: [mem 0x0000002080000000-0x000000307ff7ffff] usable [ 0.000000] reserve setup_data: [mem 0x000000307ff80000-0x000000307fffffff] reserved [ 0.000000] reserve setup_data: [mem 0x0000003080000000-0x000000407ff7ffff] usable [ 0.000000] reserve setup_data: [mem 0x000000407ff80000-0x000000407fffffff] reserved [ 0.000000] efi: EFI v2.50 by Dell Inc. [ 0.000000] efi: ACPI=0x6fffe000 ACPI 2.0=0x6fffe014 SMBIOS=0x6eab5000 SMBIOS 3.0=0x6eab3000 [ 0.000000] efi: mem00: type=3, attr=0xf, range=[0x0000000000000000-0x0000000000001000) (0MB) [ 0.000000] efi: mem01: type=2, attr=0xf, range=[0x0000000000001000-0x0000000000002000) (0MB) [ 0.000000] efi: mem02: type=7, attr=0xf, range=[0x0000000000002000-0x0000000000010000) (0MB) [ 0.000000] efi: mem03: type=3, attr=0xf, range=[0x0000000000010000-0x0000000000014000) (0MB) [ 0.000000] efi: mem04: type=7, attr=0xf, range=[0x0000000000014000-0x0000000000063000) (0MB) [ 0.000000] efi: mem05: type=3, attr=0xf, range=[0x0000000000063000-0x000000000008f000) (0MB) [ 0.000000] efi: mem06: type=10, attr=0xf, range=[0x000000000008f000-0x0000000000090000) (0MB) [ 0.000000] efi: mem07: type=3, attr=0xf, range=[0x0000000000090000-0x00000000000a0000) (0MB) [ 0.000000] efi: mem08: type=4, attr=0xf, range=[0x0000000000100000-0x0000000000120000) (0MB) [ 0.000000] efi: mem09: type=7, attr=0xf, range=[0x0000000000120000-0x0000000000c00000) (10MB) [ 0.000000] efi: mem10: type=3, attr=0xf, range=[0x0000000000c00000-0x0000000001000000) (4MB) [ 0.000000] efi: mem11: type=2, attr=0xf, range=[0x0000000001000000-0x000000000267b000) (22MB) [ 0.000000] efi: mem12: type=7, attr=0xf, range=[0x000000000267b000-0x0000000004000000) (25MB) [ 0.000000] efi: mem13: type=4, attr=0xf, range=[0x0000000004000000-0x000000000403b000) (0MB) [ 0.000000] efi: mem14: type=7, attr=0xf, range=[0x000000000403b000-0x0000000037833000) (823MB) [ 0.000000] efi: mem15: type=2, attr=0xf, range=[0x0000000037833000-0x000000004ed3b000) (373MB) [ 0.000000] efi: mem16: type=7, attr=0xf, range=[0x000000004ed3b000-0x000000004ed41000) (0MB) [ 0.000000] efi: mem17: type=1, attr=0xf, range=[0x000000004ed41000-0x000000004ee5e000) (1MB) [ 0.000000] efi: mem18: type=2, attr=0xf, range=[0x000000004ee5e000-0x000000004ef7c000) (1MB) [ 0.000000] efi: mem19: type=1, attr=0xf, range=[0x000000004ef7c000-0x000000004f08b000) (1MB) [ 0.000000] efi: mem20: type=3, attr=0xf, range=[0x000000004f08b000-0x000000004f723000) (6MB) [ 0.000000] efi: mem21: type=0, attr=0xf, range=[0x000000004f723000-0x000000005772c000) (128MB) [ 0.000000] efi: mem22: type=3, attr=0xf, range=[0x000000005772c000-0x000000005792e000) (2MB) [ 0.000000] efi: mem23: type=4, attr=0xf, range=[0x000000005792e000-0x000000005b4cf000) (59MB) [ 0.000000] efi: mem24: type=3, attr=0xf, range=[0x000000005b4cf000-0x000000005b4d0000) (0MB) [ 0.000000] efi: mem25: type=7, attr=0xf, range=[0x000000005b4d0000-0x000000005b4d2000) (0MB) [ 0.000000] efi: mem26: type=2, attr=0xf, range=[0x000000005b4d2000-0x000000005b4d6000) (0MB) [ 0.000000] efi: mem27: type=3, attr=0xf, range=[0x000000005b4d6000-0x000000005b8cf000) (3MB) [ 0.000000] efi: mem28: type=7, attr=0xf, range=[0x000000005b8cf000-0x00000000652e8000) (154MB) [ 0.000000] efi: mem29: type=4, attr=0xf, range=[0x00000000652e8000-0x000000006537b000) (0MB) [ 0.000000] efi: mem30: type=7, attr=0xf, range=[0x000000006537b000-0x000000006537f000) (0MB) [ 0.000000] efi: mem31: type=4, attr=0xf, range=[0x000000006537f000-0x000000006538e000) (0MB) [ 0.000000] efi: mem32: type=7, attr=0xf, range=[0x000000006538e000-0x0000000065390000) (0MB) [ 0.000000] efi: mem33: type=4, attr=0xf, range=[0x0000000065390000-0x0000000065420000) (0MB) [ 0.000000] efi: mem34: type=7, attr=0xf, range=[0x0000000065420000-0x0000000065421000) (0MB) [ 0.000000] efi: mem35: type=4, attr=0xf, range=[0x0000000065421000-0x0000000065734000) (3MB) [ 0.000000] efi: mem36: type=7, attr=0xf, range=[0x0000000065734000-0x0000000065736000) (0MB) [ 0.000000] efi: mem37: type=4, attr=0xf, range=[0x0000000065736000-0x0000000065737000) (0MB) [ 0.000000] efi: mem38: type=7, attr=0xf, range=[0x0000000065737000-0x0000000065739000) (0MB) [ 0.000000] efi: mem39: type=4, attr=0xf, range=[0x0000000065739000-0x000000006573b000) (0MB) [ 0.000000] efi: mem40: type=7, attr=0xf, range=[0x000000006573b000-0x000000006573d000) (0MB) [ 0.000000] efi: mem41: type=4, attr=0xf, range=[0x000000006573d000-0x000000006573f000) (0MB) [ 0.000000] efi: mem42: type=7, attr=0xf, range=[0x000000006573f000-0x0000000065740000) (0MB) [ 0.000000] efi: mem43: type=4, attr=0xf, range=[0x0000000065740000-0x0000000065741000) (0MB) [ 0.000000] efi: mem44: type=7, attr=0xf, range=[0x0000000065741000-0x0000000065743000) (0MB) [ 0.000000] efi: mem45: type=4, attr=0xf, range=[0x0000000065743000-0x0000000065745000) (0MB) [ 0.000000] efi: mem46: type=7, attr=0xf, range=[0x0000000065745000-0x0000000065746000) (0MB) [ 0.000000] efi: mem47: type=4, attr=0xf, range=[0x0000000065746000-0x000000006574b000) (0MB) [ 0.000000] efi: mem48: type=7, attr=0xf, range=[0x000000006574b000-0x000000006574d000) (0MB) [ 0.000000] efi: mem49: type=4, attr=0xf, range=[0x000000006574d000-0x000000006574e000) (0MB) [ 0.000000] efi: mem50: type=7, attr=0xf, range=[0x000000006574e000-0x000000006574f000) (0MB) [ 0.000000] efi: mem51: type=4, attr=0xf, range=[0x000000006574f000-0x0000000065752000) (0MB) [ 0.000000] efi: mem52: type=7, attr=0xf, range=[0x0000000065752000-0x0000000065754000) (0MB) [ 0.000000] efi: mem53: type=4, attr=0xf, range=[0x0000000065754000-0x0000000065756000) (0MB) [ 0.000000] efi: mem54: type=7, attr=0xf, range=[0x0000000065756000-0x0000000065759000) (0MB) [ 0.000000] efi: mem55: type=4, attr=0xf, range=[0x0000000065759000-0x000000006575a000) (0MB) [ 0.000000] efi: mem56: type=7, attr=0xf, range=[0x000000006575a000-0x000000006575b000) (0MB) [ 0.000000] efi: mem57: type=4, attr=0xf, range=[0x000000006575b000-0x0000000065776000) (0MB) [ 0.000000] efi: mem58: type=7, attr=0xf, range=[0x0000000065776000-0x0000000065778000) (0MB) [ 0.000000] efi: mem59: type=4, attr=0xf, range=[0x0000000065778000-0x000000006577a000) (0MB) [ 0.000000] efi: mem60: type=7, attr=0xf, range=[0x000000006577a000-0x000000006577c000) (0MB) [ 0.000000] efi: mem61: type=4, attr=0xf, range=[0x000000006577c000-0x000000006577e000) (0MB) [ 0.000000] efi: mem62: type=7, attr=0xf, range=[0x000000006577e000-0x0000000065780000) (0MB) [ 0.000000] efi: mem63: type=4, attr=0xf, range=[0x0000000065780000-0x0000000065782000) (0MB) [ 0.000000] efi: mem64: type=7, attr=0xf, range=[0x0000000065782000-0x0000000065785000) (0MB) [ 0.000000] efi: mem65: type=4, attr=0xf, range=[0x0000000065785000-0x0000000065787000) (0MB) [ 0.000000] efi: mem66: type=7, attr=0xf, range=[0x0000000065787000-0x0000000065789000) (0MB) [ 0.000000] efi: mem67: type=4, attr=0xf, range=[0x0000000065789000-0x000000006578b000) (0MB) [ 0.000000] efi: mem68: type=7, attr=0xf, range=[0x000000006578b000-0x000000006578d000) (0MB) [ 0.000000] efi: mem69: type=4, attr=0xf, range=[0x000000006578d000-0x000000006578f000) (0MB) [ 0.000000] efi: mem70: type=7, attr=0xf, range=[0x000000006578f000-0x0000000065790000) (0MB) [ 0.000000] efi: mem71: type=4, attr=0xf, range=[0x0000000065790000-0x0000000065795000) (0MB) [ 0.000000] efi: mem72: type=7, attr=0xf, range=[0x0000000065795000-0x0000000065796000) (0MB) [ 0.000000] efi: mem73: type=4, attr=0xf, range=[0x0000000065796000-0x0000000065799000) (0MB) [ 0.000000] efi: mem74: type=7, attr=0xf, range=[0x0000000065799000-0x000000006579b000) (0MB) [ 0.000000] efi: mem75: type=4, attr=0xf, range=[0x000000006579b000-0x000000006579e000) (0MB) [ 0.000000] efi: mem76: type=7, attr=0xf, range=[0x000000006579e000-0x000000006579f000) (0MB) [ 0.000000] efi: mem77: type=4, attr=0xf, range=[0x000000006579f000-0x000000006592c000) (1MB) [ 0.000000] efi: mem78: type=7, attr=0xf, range=[0x000000006592c000-0x000000006592d000) (0MB) [ 0.000000] efi: mem79: type=4, attr=0xf, range=[0x000000006592d000-0x000000006593d000) (0MB) [ 0.000000] efi: mem80: type=7, attr=0xf, range=[0x000000006593d000-0x000000006593e000) (0MB) [ 0.000000] efi: mem81: type=4, attr=0xf, range=[0x000000006593e000-0x0000000065947000) (0MB) [ 0.000000] efi: mem82: type=7, attr=0xf, range=[0x0000000065947000-0x0000000065948000) (0MB) [ 0.000000] efi: mem83: type=4, attr=0xf, range=[0x0000000065948000-0x000000006594d000) (0MB) [ 0.000000] efi: mem84: type=7, attr=0xf, range=[0x000000006594d000-0x000000006594e000) (0MB) [ 0.000000] efi: mem85: type=4, attr=0xf, range=[0x000000006594e000-0x0000000065952000) (0MB) [ 0.000000] efi: mem86: type=7, attr=0xf, range=[0x0000000065952000-0x0000000065953000) (0MB) [ 0.000000] efi: mem87: type=4, attr=0xf, range=[0x0000000065953000-0x0000000065955000) (0MB) [ 0.000000] efi: mem88: type=7, attr=0xf, range=[0x0000000065955000-0x0000000065956000) (0MB) [ 0.000000] efi: mem89: type=4, attr=0xf, range=[0x0000000065956000-0x000000006595a000) (0MB) [ 0.000000] efi: mem90: type=7, attr=0xf, range=[0x000000006595a000-0x000000006595b000) (0MB) [ 0.000000] efi: mem91: type=4, attr=0xf, range=[0x000000006595b000-0x0000000065974000) (0MB) [ 0.000000] efi: mem92: type=7, attr=0xf, range=[0x0000000065974000-0x0000000065976000) (0MB) [ 0.000000] efi: mem93: type=4, attr=0xf, range=[0x0000000065976000-0x000000006597a000) (0MB) [ 0.000000] efi: mem94: type=7, attr=0xf, range=[0x000000006597a000-0x000000006597b000) (0MB) [ 0.000000] efi: mem95: type=4, attr=0xf, range=[0x000000006597b000-0x0000000065982000) (0MB) [ 0.000000] efi: mem96: type=7, attr=0xf, range=[0x0000000065982000-0x0000000065983000) (0MB) [ 0.000000] efi: mem97: type=4, attr=0xf, range=[0x0000000065983000-0x000000006598f000) (0MB) [ 0.000000] efi: mem98: type=7, attr=0xf, range=[0x000000006598f000-0x0000000065990000) (0MB) [ 0.000000] efi: mem99: type=4, attr=0xf, range=[0x0000000065990000-0x0000000065997000) (0MB) [ 0.000000] efi: mem100: type=7, attr=0xf, range=[0x0000000065997000-0x0000000065998000) (0MB) [ 0.000000] efi: mem101: type=4, attr=0xf, range=[0x0000000065998000-0x00000000659a5000) (0MB) [ 0.000000] efi: mem102: type=7, attr=0xf, range=[0x00000000659a5000-0x00000000659a6000) (0MB) [ 0.000000] efi: mem103: type=4, attr=0xf, range=[0x00000000659a6000-0x0000000065cb1000) (3MB) [ 0.000000] efi: mem104: type=7, attr=0xf, range=[0x0000000065cb1000-0x0000000065cb2000) (0MB) [ 0.000000] efi: mem105: type=4, attr=0xf, range=[0x0000000065cb2000-0x0000000065d0e000) (0MB) [ 0.000000] efi: mem106: type=7, attr=0xf, range=[0x0000000065d0e000-0x0000000065d0f000) (0MB) [ 0.000000] efi: mem107: type=4, attr=0xf, range=[0x0000000065d0f000-0x0000000065d11000) (0MB) [ 0.000000] efi: mem108: type=7, attr=0xf, range=[0x0000000065d11000-0x0000000065d12000) (0MB) [ 0.000000] efi: mem109: type=4, attr=0xf, range=[0x0000000065d12000-0x0000000065d4d000) (0MB) [ 0.000000] efi: mem110: type=7, attr=0xf, range=[0x0000000065d4d000-0x0000000065d4e000) (0MB) [ 0.000000] efi: mem111: type=4, attr=0xf, range=[0x0000000065d4e000-0x0000000065d51000) (0MB) [ 0.000000] efi: mem112: type=7, attr=0xf, range=[0x0000000065d51000-0x0000000065d52000) (0MB) [ 0.000000] efi: mem113: type=4, attr=0xf, range=[0x0000000065d52000-0x0000000065d53000) (0MB) [ 0.000000] efi: mem114: type=7, attr=0xf, range=[0x0000000065d53000-0x0000000065d54000) (0MB) [ 0.000000] efi: mem115: type=4, attr=0xf, range=[0x0000000065d54000-0x0000000065d69000) (0MB) [ 0.000000] efi: mem116: type=7, attr=0xf, range=[0x0000000065d69000-0x0000000065d6a000) (0MB) [ 0.000000] efi: mem117: type=4, attr=0xf, range=[0x0000000065d6a000-0x0000000065da4000) (0MB) [ 0.000000] efi: mem118: type=7, attr=0xf, range=[0x0000000065da4000-0x0000000065da5000) (0MB) [ 0.000000] efi: mem119: type=4, attr=0xf, range=[0x0000000065da5000-0x0000000065db3000) (0MB) [ 0.000000] efi: mem120: type=7, attr=0xf, range=[0x0000000065db3000-0x0000000065db4000) (0MB) [ 0.000000] efi: mem121: type=4, attr=0xf, range=[0x0000000065db4000-0x0000000065db9000) (0MB) [ 0.000000] efi: mem122: type=7, attr=0xf, range=[0x0000000065db9000-0x0000000065dba000) (0MB) [ 0.000000] efi: mem123: type=4, attr=0xf, range=[0x0000000065dba000-0x0000000065dd2000) (0MB) [ 0.000000] efi: mem124: type=7, attr=0xf, range=[0x0000000065dd2000-0x0000000065dd4000) (0MB) [ 0.000000] efi: mem125: type=4, attr=0xf, range=[0x0000000065dd4000-0x0000000065dd9000) (0MB) [ 0.000000] efi: mem126: type=7, attr=0xf, range=[0x0000000065dd9000-0x0000000065dda000) (0MB) [ 0.000000] efi: mem127: type=4, attr=0xf, range=[0x0000000065dda000-0x0000000065de6000) (0MB) [ 0.000000] efi: mem128: type=7, attr=0xf, range=[0x0000000065de6000-0x0000000065de7000) (0MB) [ 0.000000] efi: mem129: type=4, attr=0xf, range=[0x0000000065de7000-0x0000000065de8000) (0MB) [ 0.000000] efi: mem130: type=7, attr=0xf, range=[0x0000000065de8000-0x0000000065dea000) (0MB) [ 0.000000] efi: mem131: type=4, attr=0xf, range=[0x0000000065dea000-0x0000000065e4a000) (0MB) [ 0.000000] efi: mem132: type=7, attr=0xf, range=[0x0000000065e4a000-0x0000000065e4b000) (0MB) [ 0.000000] efi: mem133: type=4, attr=0xf, range=[0x0000000065e4b000-0x0000000065e5e000) (0MB) [ 0.000000] efi: mem134: type=7, attr=0xf, range=[0x0000000065e5e000-0x0000000065e5f000) (0MB) [ 0.000000] efi: mem135: type=4, attr=0xf, range=[0x0000000065e5f000-0x0000000065e66000) (0MB) [ 0.000000] efi: mem136: type=7, attr=0xf, range=[0x0000000065e66000-0x0000000065e67000) (0MB) [ 0.000000] efi: mem137: type=4, attr=0xf, range=[0x0000000065e67000-0x0000000065e69000) (0MB) [ 0.000000] efi: mem138: type=7, attr=0xf, range=[0x0000000065e69000-0x0000000065e6a000) (0MB) [ 0.000000] efi: mem139: type=4, attr=0xf, range=[0x0000000065e6a000-0x0000000065e73000) (0MB) [ 0.000000] efi: mem140: type=7, attr=0xf, range=[0x0000000065e73000-0x0000000065e74000) (0MB) [ 0.000000] efi: mem141: type=4, attr=0xf, range=[0x0000000065e74000-0x0000000065e8e000) (0MB) [ 0.000000] efi: mem142: type=7, attr=0xf, range=[0x0000000065e8e000-0x0000000065e8f000) (0MB) [ 0.000000] efi: mem143: type=4, attr=0xf, range=[0x0000000065e8f000-0x0000000065ea0000) (0MB) [ 0.000000] efi: mem144: type=7, attr=0xf, range=[0x0000000065ea0000-0x0000000065ea1000) (0MB) [ 0.000000] efi: mem145: type=4, attr=0xf, range=[0x0000000065ea1000-0x0000000069793000) (56MB) [ 0.000000] efi: mem146: type=7, attr=0xf, range=[0x0000000069793000-0x0000000069795000) (0MB) [ 0.000000] efi: mem147: type=4, attr=0xf, range=[0x0000000069795000-0x000000006b8cf000) (33MB) [ 0.000000] efi: mem148: type=3, attr=0xf, range=[0x000000006b8cf000-0x000000006cacf000) (18MB) [ 0.000000] efi: mem149: type=6, attr=0x800000000000000f, range=[0x000000006cacf000-0x000000006cbcf000) (1MB) [ 0.000000] efi: mem150: type=5, attr=0x800000000000000f, range=[0x000000006cbcf000-0x000000006cdcf000) (2MB) [ 0.000000] efi: mem151: type=0, attr=0xf, range=[0x000000006cdcf000-0x000000006efcf000) (34MB) [ 0.000000] efi: mem152: type=10, attr=0xf, range=[0x000000006efcf000-0x000000006fdff000) (14MB) [ 0.000000] efi: mem153: type=9, attr=0xf, range=[0x000000006fdff000-0x000000006ffff000) (2MB) [ 0.000000] efi: mem154: type=4, attr=0xf, range=[0x000000006ffff000-0x0000000070000000) (0MB) [ 0.000000] efi: mem155: type=7, attr=0xf, range=[0x0000000100000000-0x000000107f380000) (63475MB) [ 0.000000] efi: mem156: type=7, attr=0xf, range=[0x0000001080000000-0x000000207ff80000) (65535MB) [ 0.000000] efi: mem157: type=7, attr=0xf, range=[0x0000002080000000-0x000000307ff80000) (65535MB) [ 0.000000] efi: mem158: type=7, attr=0xf, range=[0x0000003080000000-0x000000407ff80000) (65535MB) [ 0.000000] efi: mem159: type=0, attr=0x9, range=[0x0000000070000000-0x0000000080000000) (256MB) [ 0.000000] efi: mem160: type=11, attr=0x800000000000000f, range=[0x0000000080000000-0x0000000090000000) (256MB) [ 0.000000] efi: mem161: type=11, attr=0x800000000000000f, range=[0x00000000fec10000-0x00000000fec11000) (0MB) [ 0.000000] efi: mem162: type=11, attr=0x800000000000000f, range=[0x00000000fed80000-0x00000000fed81000) (0MB) [ 0.000000] efi: mem163: type=0, attr=0x0, range=[0x000000107f380000-0x0000001080000000) (12MB) [ 0.000000] efi: mem164: type=0, attr=0x0, range=[0x000000207ff80000-0x0000002080000000) (0MB) [ 0.000000] efi: mem165: type=0, attr=0x0, range=[0x000000307ff80000-0x0000003080000000) (0MB) [ 0.000000] efi: mem166: type=0, attr=0x0, range=[0x000000407ff80000-0x0000004080000000) (0MB) [ 0.000000] SMBIOS 3.2.0 present. [ 0.000000] DMI: Dell Inc. PowerEdge R6415/07YXFK, BIOS 1.12.2 11/15/2019 [ 0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved [ 0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable [ 0.000000] e820: last_pfn = 0x407ff80 max_arch_pfn = 0x400000000 [ 0.000000] MTRR default type: uncachable [ 0.000000] MTRR fixed ranges enabled: [ 0.000000] 00000-9FFFF write-back [ 0.000000] A0000-FFFFF uncachable [ 0.000000] MTRR variable ranges enabled: [ 0.000000] 0 base 0000FF000000 mask FFFFFF000000 write-protect [ 0.000000] 1 base 000000000000 mask FFFF80000000 write-back [ 0.000000] 2 base 000070000000 mask FFFFF0000000 uncachable [ 0.000000] 3 disabled [ 0.000000] 4 disabled [ 0.000000] 5 disabled [ 0.000000] 6 disabled [ 0.000000] 7 disabled [ 0.000000] TOM2: 0000004080000000 aka 264192M [ 0.000000] PAT configuration [0-7]: WB WC UC- UC WB WP UC- UC [ 0.000000] e820: last_pfn = 0x70000 max_arch_pfn = 0x400000000 [ 0.000000] Base memory trampoline at [ffff8ebbc0099000] 99000 size 24576 [ 0.000000] Using GB pages for direct mapping [ 0.000000] BRK [0xfac853000, 0xfac853fff] PGTABLE [ 0.000000] BRK [0xfac854000, 0xfac854fff] PGTABLE [ 0.000000] BRK [0xfac855000, 0xfac855fff] PGTABLE [ 0.000000] BRK [0xfac856000, 0xfac856fff] PGTABLE [ 0.000000] BRK [0xfac857000, 0xfac857fff] PGTABLE [ 0.000000] BRK [0xfac858000, 0xfac858fff] PGTABLE [ 0.000000] BRK [0xfac859000, 0xfac859fff] PGTABLE [ 0.000000] BRK [0xfac85a000, 0xfac85afff] PGTABLE [ 0.000000] BRK [0xfac85b000, 0xfac85bfff] PGTABLE [ 0.000000] BRK [0xfac85c000, 0xfac85cfff] PGTABLE [ 0.000000] BRK [0xfac85d000, 0xfac85dfff] PGTABLE [ 0.000000] BRK [0xfac85e000, 0xfac85efff] PGTABLE [ 0.000000] RAMDISK: [mem 0x37981000-0x38ca4fff] [ 0.000000] Early table checksum verification disabled [ 0.000000] ACPI: RSDP 000000006fffe014 00024 (v02 DELL ) [ 0.000000] ACPI: XSDT 000000006fffd0e8 000AC (v01 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: FACP 000000006fff0000 00114 (v06 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: DSDT 000000006ffe1000 0BD00 (v02 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: FACS 000000006fdd3000 00040 [ 0.000000] ACPI: SSDT 000000006fffc000 000D2 (v02 DELL PE_SC3 00000002 MSFT 04000000) [ 0.000000] ACPI: BERT 000000006fffb000 00030 (v01 DELL BERT 00000001 DELL 00000001) [ 0.000000] ACPI: HEST 000000006fffa000 006DC (v01 DELL HEST 00000001 DELL 00000001) [ 0.000000] ACPI: SSDT 000000006fff9000 00294 (v01 DELL PE_SC3 00000001 AMD 00000001) [ 0.000000] ACPI: SRAT 000000006fff8000 00420 (v03 DELL PE_SC3 00000001 AMD 00000001) [ 0.000000] ACPI: MSCT 000000006fff7000 0004E (v01 DELL PE_SC3 00000000 AMD 00000001) [ 0.000000] ACPI: SLIT 000000006fff6000 0003C (v01 DELL PE_SC3 00000001 AMD 00000001) [ 0.000000] ACPI: CRAT 000000006fff3000 02DC0 (v01 DELL PE_SC3 00000001 AMD 00000001) [ 0.000000] ACPI: EINJ 000000006fff2000 00150 (v01 DELL PE_SC3 00000001 AMD 00000001) [ 0.000000] ACPI: SLIC 000000006fff1000 00024 (v01 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: HPET 000000006ffef000 00038 (v01 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: APIC 000000006ffee000 004B2 (v03 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: MCFG 000000006ffed000 0003C (v01 DELL PE_SC3 00000002 DELL 00000001) [ 0.000000] ACPI: SSDT 000000006ffe0000 00629 (v02 DELL xhc_port 00000001 INTL 20170119) [ 0.000000] ACPI: IVRS 000000006ffdf000 00210 (v02 DELL PE_SC3 00000001 AMD 00000000) [ 0.000000] ACPI: SSDT 000000006ffdd000 01658 (v01 AMD CPMCMN 00000001 INTL 20170119) [ 0.000000] ACPI: Local APIC address 0xfee00000 [ 0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x01 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x02 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x03 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x04 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x05 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x08 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x09 -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x0a -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x0b -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x0c -> Node 0 [ 0.000000] SRAT: PXM 0 -> APIC 0x0d -> Node 0 [ 0.000000] SRAT: PXM 1 -> APIC 0x10 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x11 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x12 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x13 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x14 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x15 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x18 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x19 -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x1a -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x1b -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x1c -> Node 1 [ 0.000000] SRAT: PXM 1 -> APIC 0x1d -> Node 1 [ 0.000000] SRAT: PXM 2 -> APIC 0x20 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x21 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x22 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x23 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x24 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x25 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x28 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x29 -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x2a -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x2b -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x2c -> Node 2 [ 0.000000] SRAT: PXM 2 -> APIC 0x2d -> Node 2 [ 0.000000] SRAT: PXM 3 -> APIC 0x30 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x31 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x32 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x33 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x34 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x35 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x38 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x39 -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x3a -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x3b -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x3c -> Node 3 [ 0.000000] SRAT: PXM 3 -> APIC 0x3d -> Node 3 [ 0.000000] SRAT: Node 0 PXM 0 [mem 0x00000000-0x0009ffff] [ 0.000000] SRAT: Node 0 PXM 0 [mem 0x00100000-0x7fffffff] [ 0.000000] SRAT: Node 0 PXM 0 [mem 0x100000000-0x107fffffff] [ 0.000000] SRAT: Node 1 PXM 1 [mem 0x1080000000-0x207fffffff] [ 0.000000] SRAT: Node 2 PXM 2 [mem 0x2080000000-0x307fffffff] [ 0.000000] SRAT: Node 3 PXM 3 [mem 0x3080000000-0x407fffffff] [ 0.000000] NUMA: Initialized distance table, cnt=4 [ 0.000000] NUMA: Node 0 [mem 0x00000000-0x0009ffff] + [mem 0x00100000-0x7fffffff] -> [mem 0x00000000-0x7fffffff] [ 0.000000] NUMA: Node 0 [mem 0x00000000-0x7fffffff] + [mem 0x100000000-0x107fffffff] -> [mem 0x00000000-0x107fffffff] [ 0.000000] NODE_DATA(0) allocated [mem 0x107f359000-0x107f37ffff] [ 0.000000] NODE_DATA(1) allocated [mem 0x207ff59000-0x207ff7ffff] [ 0.000000] NODE_DATA(2) allocated [mem 0x307ff59000-0x307ff7ffff] [ 0.000000] NODE_DATA(3) allocated [mem 0x407ff58000-0x407ff7efff] [ 0.000000] Reserving 176MB of memory at 704MB for crashkernel (System RAM: 261692MB) [ 0.000000] Zone ranges: [ 0.000000] DMA [mem 0x00001000-0x00ffffff] [ 0.000000] DMA32 [mem 0x01000000-0xffffffff] [ 0.000000] Normal [mem 0x100000000-0x407ff7ffff] [ 0.000000] Movable zone start for each node [ 0.000000] Early memory node ranges [ 0.000000] node 0: [mem 0x00001000-0x0008efff] [ 0.000000] node 0: [mem 0x00090000-0x0009ffff] [ 0.000000] node 0: [mem 0x00100000-0x4f722fff] [ 0.000000] node 0: [mem 0x5772c000-0x6cacefff] [ 0.000000] node 0: [mem 0x6ffff000-0x6fffffff] [ 0.000000] node 0: [mem 0x100000000-0x107f37ffff] [ 0.000000] node 1: [mem 0x1080000000-0x207ff7ffff] [ 0.000000] node 2: [mem 0x2080000000-0x307ff7ffff] [ 0.000000] node 3: [mem 0x3080000000-0x407ff7ffff] [ 0.000000] Initmem setup node 0 [mem 0x00001000-0x107f37ffff] [ 0.000000] On node 0 totalpages: 16661989 [ 0.000000] DMA zone: 64 pages used for memmap [ 0.000000] DMA zone: 1126 pages reserved [ 0.000000] DMA zone: 3998 pages, LIFO batch:0 [ 0.000000] DMA32 zone: 6380 pages used for memmap [ 0.000000] DMA32 zone: 408263 pages, LIFO batch:31 [ 0.000000] Normal zone: 253902 pages used for memmap [ 0.000000] Normal zone: 16249728 pages, LIFO batch:31 [ 0.000000] Initmem setup node 1 [mem 0x1080000000-0x207ff7ffff] [ 0.000000] On node 1 totalpages: 16777088 [ 0.000000] Normal zone: 262142 pages used for memmap [ 0.000000] Normal zone: 16777088 pages, LIFO batch:31 [ 0.000000] Initmem setup node 2 [mem 0x2080000000-0x307ff7ffff] [ 0.000000] On node 2 totalpages: 16777088 [ 0.000000] Normal zone: 262142 pages used for memmap [ 0.000000] Normal zone: 16777088 pages, LIFO batch:31 [ 0.000000] Initmem setup node 3 [mem 0x3080000000-0x407ff7ffff] [ 0.000000] On node 3 totalpages: 16777088 [ 0.000000] Normal zone: 262142 pages used for memmap [ 0.000000] Normal zone: 16777088 pages, LIFO batch:31 [ 0.000000] ACPI: PM-Timer IO Port: 0x408 [ 0.000000] ACPI: Local APIC address 0xfee00000 [ 0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x10] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x20] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x30] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x04] lapic_id[0x08] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x05] lapic_id[0x18] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x06] lapic_id[0x28] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x07] lapic_id[0x38] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x08] lapic_id[0x02] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x09] lapic_id[0x12] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x22] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x32] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x0a] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x1a] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x2a] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x3a] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x10] lapic_id[0x04] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x11] lapic_id[0x14] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x12] lapic_id[0x24] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x13] lapic_id[0x34] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x14] lapic_id[0x0c] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x15] lapic_id[0x1c] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x16] lapic_id[0x2c] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x17] lapic_id[0x3c] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x18] lapic_id[0x01] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x19] lapic_id[0x11] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x1a] lapic_id[0x21] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x1b] lapic_id[0x31] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x1c] lapic_id[0x09] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x1d] lapic_id[0x19] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x1e] lapic_id[0x29] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x1f] lapic_id[0x39] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x20] lapic_id[0x03] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x21] lapic_id[0x13] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x22] lapic_id[0x23] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x23] lapic_id[0x33] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x24] lapic_id[0x0b] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x25] lapic_id[0x1b] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x26] lapic_id[0x2b] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x27] lapic_id[0x3b] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x28] lapic_id[0x05] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x29] lapic_id[0x15] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x2a] lapic_id[0x25] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x2b] lapic_id[0x35] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x2c] lapic_id[0x0d] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x2d] lapic_id[0x1d] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x2e] lapic_id[0x2d] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x2f] lapic_id[0x3d] enabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x30] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x31] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x32] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x33] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x34] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x35] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x36] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x37] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x38] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x39] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x3a] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x3b] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x3c] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x3d] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x3e] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x3f] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x40] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x41] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x42] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x43] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x44] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x45] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x46] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x47] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x48] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x49] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x4a] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x4b] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x4c] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x4d] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x4e] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x4f] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x50] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x51] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x52] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x53] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x54] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x55] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x56] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x57] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x58] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x59] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x5a] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x5b] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x5c] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x5d] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x5e] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x5f] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x60] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x61] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x62] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x63] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x64] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x65] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x66] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x67] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x68] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x69] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x6a] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x6b] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x6c] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x6d] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x6e] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x6f] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x70] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x71] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x72] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x73] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x74] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x75] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x76] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x77] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x78] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x79] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x7a] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x7b] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x7c] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x7d] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x7e] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC (acpi_id[0x7f] lapic_id[0x00] disabled) [ 0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] high edge lint[0x1]) [ 0.000000] ACPI: IOAPIC (id[0x80] address[0xfec00000] gsi_base[0]) [ 0.000000] IOAPIC[0]: apic_id 128, version 33, address 0xfec00000, GSI 0-23 [ 0.000000] ACPI: IOAPIC (id[0x81] address[0xfd880000] gsi_base[24]) [ 0.000000] IOAPIC[1]: apic_id 129, version 33, address 0xfd880000, GSI 24-55 [ 0.000000] ACPI: IOAPIC (id[0x82] address[0xe0900000] gsi_base[56]) [ 0.000000] IOAPIC[2]: apic_id 130, version 33, address 0xe0900000, GSI 56-87 [ 0.000000] ACPI: IOAPIC (id[0x83] address[0xc5900000] gsi_base[88]) [ 0.000000] IOAPIC[3]: apic_id 131, version 33, address 0xc5900000, GSI 88-119 [ 0.000000] ACPI: IOAPIC (id[0x84] address[0xaa900000] gsi_base[120]) [ 0.000000] IOAPIC[4]: apic_id 132, version 33, address 0xaa900000, GSI 120-151 [ 0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl) [ 0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 low level) [ 0.000000] ACPI: IRQ0 used by override. [ 0.000000] ACPI: IRQ9 used by override. [ 0.000000] Using ACPI (MADT) for SMP configuration information [ 0.000000] ACPI: HPET id: 0x10228201 base: 0xfed00000 [ 0.000000] smpboot: Allowing 128 CPUs, 80 hotplug CPUs [ 0.000000] PM: Registered nosave memory: [mem 0x0008f000-0x0008ffff] [ 0.000000] PM: Registered nosave memory: [mem 0x000a0000-0x000fffff] [ 0.000000] PM: Registered nosave memory: [mem 0x37833000-0x37833fff] [ 0.000000] PM: Registered nosave memory: [mem 0x3784b000-0x3784bfff] [ 0.000000] PM: Registered nosave memory: [mem 0x3784c000-0x3784cfff] [ 0.000000] PM: Registered nosave memory: [mem 0x37871000-0x37871fff] [ 0.000000] PM: Registered nosave memory: [mem 0x37872000-0x37872fff] [ 0.000000] PM: Registered nosave memory: [mem 0x3787a000-0x3787afff] [ 0.000000] PM: Registered nosave memory: [mem 0x3787b000-0x3787bfff] [ 0.000000] PM: Registered nosave memory: [mem 0x378ac000-0x378acfff] [ 0.000000] PM: Registered nosave memory: [mem 0x378ad000-0x378adfff] [ 0.000000] PM: Registered nosave memory: [mem 0x378de000-0x378defff] [ 0.000000] PM: Registered nosave memory: [mem 0x378df000-0x378dffff] [ 0.000000] PM: Registered nosave memory: [mem 0x37980000-0x37980fff] [ 0.000000] PM: Registered nosave memory: [mem 0x4f723000-0x5772bfff] [ 0.000000] PM: Registered nosave memory: [mem 0x6cacf000-0x6efcefff] [ 0.000000] PM: Registered nosave memory: [mem 0x6efcf000-0x6fdfefff] [ 0.000000] PM: Registered nosave memory: [mem 0x6fdff000-0x6fffefff] [ 0.000000] PM: Registered nosave memory: [mem 0x70000000-0x8fffffff] [ 0.000000] PM: Registered nosave memory: [mem 0x90000000-0xfec0ffff] [ 0.000000] PM: Registered nosave memory: [mem 0xfec10000-0xfec10fff] [ 0.000000] PM: Registered nosave memory: [mem 0xfec11000-0xfed7ffff] [ 0.000000] PM: Registered nosave memory: [mem 0xfed80000-0xfed80fff] [ 0.000000] PM: Registered nosave memory: [mem 0xfed81000-0xffffffff] [ 0.000000] PM: Registered nosave memory: [mem 0x107f380000-0x107fffffff] [ 0.000000] PM: Registered nosave memory: [mem 0x207ff80000-0x207fffffff] [ 0.000000] PM: Registered nosave memory: [mem 0x307ff80000-0x307fffffff] [ 0.000000] e820: [mem 0x90000000-0xfec0ffff] available for PCI devices [ 0.000000] Booting paravirtualized kernel on bare hardware [ 0.000000] setup_percpu: NR_CPUS:5120 nr_cpumask_bits:128 nr_cpu_ids:128 nr_node_ids:4 [ 0.000000] PERCPU: Embedded 38 pages/cpu @ffff8ecbfee00000 s118784 r8192 d28672 u262144 [ 0.000000] pcpu-alloc: s118784 r8192 d28672 u262144 alloc=1*2097152 [ 0.000000] pcpu-alloc: [0] 000 004 008 012 016 020 024 028 [ 0.000000] pcpu-alloc: [0] 032 036 040 044 048 052 056 060 [ 0.000000] pcpu-alloc: [0] 064 068 072 076 080 084 088 092 [ 0.000000] pcpu-alloc: [0] 096 100 104 108 112 116 120 124 [ 0.000000] pcpu-alloc: [1] 001 005 009 013 017 021 025 029 [ 0.000000] pcpu-alloc: [1] 033 037 041 045 049 053 057 061 [ 0.000000] pcpu-alloc: [1] 065 069 073 077 081 085 089 093 [ 0.000000] pcpu-alloc: [1] 097 101 105 109 113 117 121 125 [ 0.000000] pcpu-alloc: [2] 002 006 010 014 018 022 026 030 [ 0.000000] pcpu-alloc: [2] 034 038 042 046 050 054 058 062 [ 0.000000] pcpu-alloc: [2] 066 070 074 078 082 086 090 094 [ 0.000000] pcpu-alloc: [2] 098 102 106 110 114 118 122 126 [ 0.000000] pcpu-alloc: [3] 003 007 011 015 019 023 027 031 [ 0.000000] pcpu-alloc: [3] 035 039 043 047 051 055 059 063 [ 0.000000] pcpu-alloc: [3] 067 071 075 079 083 087 091 095 [ 0.000000] pcpu-alloc: [3] 099 103 107 111 115 119 123 127 [ 0.000000] Built 4 zonelists in Zone order, mobility grouping on. Total pages: 65945355 [ 0.000000] Policy zone: Normal [ 0.000000] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-3.10.0-957.27.2.el7_lustre.pl2.x86_64 root=UUID=13d7db90-76a6-4160-92a3-3d6e156edf61 ro crashkernel=auto nomodeset console=ttyS0,115200 LANG=en_US.UTF-8 [ 0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes) [ 0.000000] x86/fpu: xstate_offset[2]: 0240, xstate_sizes[2]: 0100 [ 0.000000] xsave: enabled xstate_bv 0x7, cntxt size 0x340 using standard form [ 0.000000] Memory: 9569784k/270532096k available (7676k kernel code, 2559084k absent, 4698172k reserved, 6045k data, 1876k init) [ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=128, Nodes=4 [ 0.000000] Hierarchical RCU implementation. [ 0.000000] RCU restricting CPUs from NR_CPUS=5120 to nr_cpu_ids=128. [ 0.000000] NR_IRQS:327936 nr_irqs:3624 0 [ 0.000000] Console: colour dummy device 80x25 [ 0.000000] console [ttyS0] enabled [ 0.000000] allocated 1072693248 bytes of page_cgroup [ 0.000000] please try 'cgroup_disable=memory' option if you don't want memory cgroups [ 0.000000] Enabling automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl [ 0.000000] hpet clockevent registered [ 0.000000] tsc: Fast TSC calibration using PIT [ 0.000000] tsc: Detected 1996.199 MHz processor [ 0.000053] Calibrating delay loop (skipped), value calculated using timer frequency.. 3992.39 BogoMIPS (lpj=1996199) [ 0.010696] pid_max: default: 131072 minimum: 1024 [ 0.016361] Security Framework initialized [ 0.020476] SELinux: Initializing. [ 0.024040] SELinux: Starting in permissive mode [ 0.024041] Yama: becoming mindful. [ 0.044117] Dentry cache hash table entries: 33554432 (order: 16, 268435456 bytes) [ 0.099983] Inode-cache hash table entries: 16777216 (order: 15, 134217728 bytes) [ 0.127567] Mount-cache hash table entries: 524288 (order: 10, 4194304 bytes) [ 0.134973] Mountpoint-cache hash table entries: 524288 (order: 10, 4194304 bytes) [ 0.144040] Initializing cgroup subsys memory [ 0.148423] spurious 8259A interrupt: IRQ7. [ 0.148457] Initializing cgroup subsys devices [ 0.152914] Initializing cgroup subsys freezer [ 0.157367] Initializing cgroup subsys net_cls [ 0.161823] Initializing cgroup subsys blkio [ 0.166104] Initializing cgroup subsys perf_event [ 0.170827] Initializing cgroup subsys hugetlb [ 0.175282] Initializing cgroup subsys pids [ 0.179476] Initializing cgroup subsys net_prio [ 0.184084] tseg: 0070000000 [ 0.189709] LVT offset 2 assigned for vector 0xf4 [ 0.194439] Last level iTLB entries: 4KB 1024, 2MB 1024, 4MB 512 [ 0.200459] Last level dTLB entries: 4KB 1536, 2MB 1536, 4MB 768 [ 0.206474] tlb_flushall_shift: 6 [ 0.209822] Speculative Store Bypass: Mitigation: Speculative Store Bypass disabled via prctl and seccomp [ 0.219388] FEATURE SPEC_CTRL Not Present [ 0.223408] FEATURE IBPB_SUPPORT Present [ 0.227362] Spectre V2 : Enabling Indirect Branch Prediction Barrier [ 0.233795] Spectre V2 : Mitigation: Full retpoline [ 0.239647] Freeing SMP alternatives: 28k freed [ 0.245858] ACPI: Core revision 20130517 [ 0.254801] ACPI: All ACPI Tables successfully acquired [ 0.266438] ftrace: allocating 29216 entries in 115 pages [ 0.606032] Switched APIC routing to physical flat. [ 0.612961] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1 [ 0.628969] smpboot: CPU0: AMD EPYC 7401P 24-Core Processor (fam: 17, model: 01, stepping: 02) [ 0.711416] random: fast init done [ 0.741416] APIC calibration not consistent with PM-Timer: 101ms instead of 100ms [ 0.748893] APIC delta adjusted to PM-Timer: 623827 (636297) [ 0.754582] Performance Events: Fam17h core perfctr, AMD PMU driver. [ 0.761012] ... version: 0 [ 0.765022] ... bit width: 48 [ 0.769121] ... generic registers: 6 [ 0.773135] ... value mask: 0000ffffffffffff [ 0.778449] ... max period: 00007fffffffffff [ 0.783762] ... fixed-purpose events: 0 [ 0.787773] ... event mask: 000000000000003f [ 0.796027] NMI watchdog: enabled on all CPUs, permanently consumes one hw-PMU counter. [ 0.804103] smpboot: Booting Node 1, Processors #1 OK [ 0.817309] smpboot: Booting Node 2, Processors #2 OK [ 0.830509] smpboot: Booting Node 3, Processors #3 OK [ 0.843705] smpboot: Booting Node 0, Processors #4 OK [ 0.856887] smpboot: Booting Node 1, Processors #5 OK [ 0.870066] smpboot: Booting Node 2, Processors #6 OK [ 0.883249] smpboot: Booting Node 3, Processors #7 OK [ 0.896431] smpboot: Booting Node 0, Processors #8 OK [ 0.909824] smpboot: Booting Node 1, Processors #9 OK [ 0.923023] smpboot: Booting Node 2, Processors #10 OK [ 0.936298] smpboot: Booting Node 3, Processors #11 OK [ 0.949569] smpboot: Booting Node 0, Processors #12 OK [ 0.962841] smpboot: Booting Node 1, Processors #13 OK [ 0.976115] smpboot: Booting Node 2, Processors #14 OK [ 0.989383] smpboot: Booting Node 3, Processors #15 OK [ 1.002655] smpboot: Booting Node 0, Processors #16 OK [ 1.016030] smpboot: Booting Node 1, Processors #17 OK [ 1.029313] smpboot: Booting Node 2, Processors #18 OK [ 1.042594] smpboot: Booting Node 3, Processors #19 OK [ 1.055863] smpboot: Booting Node 0, Processors #20 OK [ 1.069128] smpboot: Booting Node 1, Processors #21 OK [ 1.082398] smpboot: Booting Node 2, Processors #22 OK [ 1.095680] smpboot: Booting Node 3, Processors #23 OK [ 1.108947] smpboot: Booting Node 0, Processors #24 OK [ 1.122668] smpboot: Booting Node 1, Processors #25 OK [ 1.135919] smpboot: Booting Node 2, Processors #26 OK [ 1.149158] smpboot: Booting Node 3, Processors #27 OK [ 1.162384] smpboot: Booting Node 0, Processors #28 OK [ 1.175614] smpboot: Booting Node 1, Processors #29 OK [ 1.188848] smpboot: Booting Node 2, Processors #30 OK [ 1.202080] smpboot: Booting Node 3, Processors #31 OK [ 1.215297] smpboot: Booting Node 0, Processors #32 OK [ 1.228631] smpboot: Booting Node 1, Processors #33 OK [ 1.241865] smpboot: Booting Node 2, Processors #34 OK [ 1.255208] smpboot: Booting Node 3, Processors #35 OK [ 1.268433] smpboot: Booting Node 0, Processors #36 OK [ 1.281662] smpboot: Booting Node 1, Processors #37 OK [ 1.294906] smpboot: Booting Node 2, Processors #38 OK [ 1.308154] smpboot: Booting Node 3, Processors #39 OK [ 1.321381] smpboot: Booting Node 0, Processors #40 OK [ 1.334712] smpboot: Booting Node 1, Processors #41 OK [ 1.348050] smpboot: Booting Node 2, Processors #42 OK [ 1.361386] smpboot: Booting Node 3, Processors #43 OK [ 1.374612] smpboot: Booting Node 0, Processors #44 OK [ 1.387842] smpboot: Booting Node 1, Processors #45 OK [ 1.401081] smpboot: Booting Node 2, Processors #46 OK [ 1.414410] smpboot: Booting Node 3, Processors #47 [ 1.427122] Brought up 48 CPUs [ 1.430379] smpboot: Max logical packages: 3 [ 1.434655] smpboot: Total of 48 processors activated (191635.10 BogoMIPS) [ 1.724563] node 0 initialised, 15458277 pages in 275ms [ 1.731864] node 2 initialised, 15989367 pages in 278ms [ 1.731882] node 1 initialised, 15989367 pages in 278ms [ 1.733842] node 3 initialised, 15989249 pages in 280ms [ 1.748087] devtmpfs: initialized [ 1.773772] EVM: security.selinux [ 1.777092] EVM: security.ima [ 1.780063] EVM: security.capability [ 1.783735] PM: Registering ACPI NVS region [mem 0x0008f000-0x0008ffff] (4096 bytes) [ 1.791481] PM: Registering ACPI NVS region [mem 0x6efcf000-0x6fdfefff] (14876672 bytes) [ 1.801099] atomic64 test passed for x86-64 platform with CX8 and with SSE [ 1.807974] pinctrl core: initialized pinctrl subsystem [ 1.813298] RTC time: 16:23:52, date: 07/08/20 [ 1.817897] NET: Registered protocol family 16 [ 1.822694] ACPI FADT declares the system doesn't support PCIe ASPM, so disable it [ 1.830264] ACPI: bus type PCI registered [ 1.834277] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5 [ 1.840854] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0x80000000-0x8fffffff] (base 0x80000000) [ 1.850155] PCI: MMCONFIG at [mem 0x80000000-0x8fffffff] reserved in E820 [ 1.856947] PCI: Using configuration type 1 for base access [ 1.862532] PCI: Dell System detected, enabling pci=bfsort. [ 1.878907] ACPI: Added _OSI(Module Device) [ 1.883099] ACPI: Added _OSI(Processor Device) [ 1.887543] ACPI: Added _OSI(3.0 _SCP Extensions) [ 1.892247] ACPI: Added _OSI(Processor Aggregator Device) [ 1.897649] ACPI: Added _OSI(Linux-Dell-Video) [ 1.902953] ACPI: EC: Look up EC in DSDT [ 1.903994] ACPI: Executed 2 blocks of module-level executable AML code [ 1.916069] ACPI: Interpreter enabled [ 1.919749] ACPI: (supports S0 S5) [ 1.923154] ACPI: Using IOAPIC for interrupt routing [ 1.928332] HEST: Table parsing has been initialized. [ 1.933384] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug [ 1.942532] ACPI: Enabled 1 GPEs in block 00 to 1F [ 1.955032] ACPI: PCI Interrupt Link [LNKA] (IRQs 4 5 7 10 11 14 15) *0 [ 1.961941] ACPI: PCI Interrupt Link [LNKB] (IRQs 4 5 7 10 11 14 15) *0 [ 1.968847] ACPI: PCI Interrupt Link [LNKC] (IRQs 4 5 7 10 11 14 15) *0 [ 1.975755] ACPI: PCI Interrupt Link [LNKD] (IRQs 4 5 7 10 11 14 15) *0 [ 1.982663] ACPI: PCI Interrupt Link [LNKE] (IRQs 4 5 7 10 11 14 15) *0 [ 1.989571] ACPI: PCI Interrupt Link [LNKF] (IRQs 4 5 7 10 11 14 15) *0 [ 1.996478] ACPI: PCI Interrupt Link [LNKG] (IRQs 4 5 7 10 11 14 15) *0 [ 2.003385] ACPI: PCI Interrupt Link [LNKH] (IRQs 4 5 7 10 11 14 15) *0 [ 2.010437] ACPI: PCI Root Bridge [PC00] (domain 0000 [bus 00-3f]) [ 2.016623] acpi PNP0A08:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI] [ 2.024835] acpi PNP0A08:00: PCIe AER handled by firmware [ 2.030278] acpi PNP0A08:00: _OSC: platform does not support [SHPCHotplug] [ 2.037225] acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug PME PCIeCapability] [ 2.044875] acpi PNP0A08:00: FADT indicates ASPM is unsupported, using BIOS configuration [ 2.053492] PCI host bridge to bus 0000:00 [ 2.057591] pci_bus 0000:00: root bus resource [io 0x0000-0x03af window] [ 2.064376] pci_bus 0000:00: root bus resource [io 0x03e0-0x0cf7 window] [ 2.071161] pci_bus 0000:00: root bus resource [mem 0x000c0000-0x000c3fff window] [ 2.078641] pci_bus 0000:00: root bus resource [mem 0x000c4000-0x000c7fff window] [ 2.086120] pci_bus 0000:00: root bus resource [mem 0x000c8000-0x000cbfff window] [ 2.093602] pci_bus 0000:00: root bus resource [mem 0x000cc000-0x000cffff window] [ 2.101080] pci_bus 0000:00: root bus resource [mem 0x000d0000-0x000d3fff window] [ 2.108559] pci_bus 0000:00: root bus resource [mem 0x000d4000-0x000d7fff window] [ 2.116040] pci_bus 0000:00: root bus resource [mem 0x000d8000-0x000dbfff window] [ 2.123518] pci_bus 0000:00: root bus resource [mem 0x000dc000-0x000dffff window] [ 2.130999] pci_bus 0000:00: root bus resource [mem 0x000e0000-0x000e3fff window] [ 2.138478] pci_bus 0000:00: root bus resource [mem 0x000e4000-0x000e7fff window] [ 2.145957] pci_bus 0000:00: root bus resource [mem 0x000e8000-0x000ebfff window] [ 2.153438] pci_bus 0000:00: root bus resource [mem 0x000ec000-0x000effff window] [ 2.160917] pci_bus 0000:00: root bus resource [mem 0x000f0000-0x000fffff window] [ 2.168397] pci_bus 0000:00: root bus resource [io 0x0d00-0x3fff window] [ 2.175182] pci_bus 0000:00: root bus resource [mem 0xe1000000-0xfebfffff window] [ 2.182661] pci_bus 0000:00: root bus resource [mem 0x10000000000-0x2bf3fffffff window] [ 2.190663] pci_bus 0000:00: root bus resource [bus 00-3f] [ 2.196154] pci 0000:00:00.0: [1022:1450] type 00 class 0x060000 [ 2.196234] pci 0000:00:00.2: [1022:1451] type 00 class 0x080600 [ 2.196322] pci 0000:00:01.0: [1022:1452] type 00 class 0x060000 [ 2.196399] pci 0000:00:02.0: [1022:1452] type 00 class 0x060000 [ 2.196473] pci 0000:00:03.0: [1022:1452] type 00 class 0x060000 [ 2.196537] pci 0000:00:03.1: [1022:1453] type 01 class 0x060400 [ 2.196684] pci 0000:00:03.1: PME# supported from D0 D3hot D3cold [ 2.196785] pci 0000:00:04.0: [1022:1452] type 00 class 0x060000 [ 2.196866] pci 0000:00:07.0: [1022:1452] type 00 class 0x060000 [ 2.196927] pci 0000:00:07.1: [1022:1454] type 01 class 0x060400 [ 2.197696] pci 0000:00:07.1: PME# supported from D0 D3hot D3cold [ 2.197775] pci 0000:00:08.0: [1022:1452] type 00 class 0x060000 [ 2.197837] pci 0000:00:08.1: [1022:1454] type 01 class 0x060400 [ 2.198696] pci 0000:00:08.1: PME# supported from D0 D3hot D3cold [ 2.198811] pci 0000:00:14.0: [1022:790b] type 00 class 0x0c0500 [ 2.199012] pci 0000:00:14.3: [1022:790e] type 00 class 0x060100 [ 2.199218] pci 0000:00:18.0: [1022:1460] type 00 class 0x060000 [ 2.199270] pci 0000:00:18.1: [1022:1461] type 00 class 0x060000 [ 2.199321] pci 0000:00:18.2: [1022:1462] type 00 class 0x060000 [ 2.199371] pci 0000:00:18.3: [1022:1463] type 00 class 0x060000 [ 2.199423] pci 0000:00:18.4: [1022:1464] type 00 class 0x060000 [ 2.199473] pci 0000:00:18.5: [1022:1465] type 00 class 0x060000 [ 2.199527] pci 0000:00:18.6: [1022:1466] type 00 class 0x060000 [ 2.199577] pci 0000:00:18.7: [1022:1467] type 00 class 0x060000 [ 2.199628] pci 0000:00:19.0: [1022:1460] type 00 class 0x060000 [ 2.199682] pci 0000:00:19.1: [1022:1461] type 00 class 0x060000 [ 2.199737] pci 0000:00:19.2: [1022:1462] type 00 class 0x060000 [ 2.199789] pci 0000:00:19.3: [1022:1463] type 00 class 0x060000 [ 2.199843] pci 0000:00:19.4: [1022:1464] type 00 class 0x060000 [ 2.199895] pci 0000:00:19.5: [1022:1465] type 00 class 0x060000 [ 2.199950] pci 0000:00:19.6: [1022:1466] type 00 class 0x060000 [ 2.200002] pci 0000:00:19.7: [1022:1467] type 00 class 0x060000 [ 2.200057] pci 0000:00:1a.0: [1022:1460] type 00 class 0x060000 [ 2.200109] pci 0000:00:1a.1: [1022:1461] type 00 class 0x060000 [ 2.200162] pci 0000:00:1a.2: [1022:1462] type 00 class 0x060000 [ 2.200215] pci 0000:00:1a.3: [1022:1463] type 00 class 0x060000 [ 2.200271] pci 0000:00:1a.4: [1022:1464] type 00 class 0x060000 [ 2.200323] pci 0000:00:1a.5: [1022:1465] type 00 class 0x060000 [ 2.200378] pci 0000:00:1a.6: [1022:1466] type 00 class 0x060000 [ 2.200431] pci 0000:00:1a.7: [1022:1467] type 00 class 0x060000 [ 2.200485] pci 0000:00:1b.0: [1022:1460] type 00 class 0x060000 [ 2.200540] pci 0000:00:1b.1: [1022:1461] type 00 class 0x060000 [ 2.200597] pci 0000:00:1b.2: [1022:1462] type 00 class 0x060000 [ 2.200650] pci 0000:00:1b.3: [1022:1463] type 00 class 0x060000 [ 2.200705] pci 0000:00:1b.4: [1022:1464] type 00 class 0x060000 [ 2.200759] pci 0000:00:1b.5: [1022:1465] type 00 class 0x060000 [ 2.200813] pci 0000:00:1b.6: [1022:1466] type 00 class 0x060000 [ 2.200866] pci 0000:00:1b.7: [1022:1467] type 00 class 0x060000 [ 2.201722] pci 0000:01:00.0: [15b3:101b] type 00 class 0x020700 [ 2.201869] pci 0000:01:00.0: reg 0x10: [mem 0xe2000000-0xe3ffffff 64bit pref] [ 2.202103] pci 0000:01:00.0: reg 0x30: [mem 0xfff00000-0xffffffff pref] [ 2.202509] pci 0000:01:00.0: PME# supported from D3cold [ 2.202788] pci 0000:00:03.1: PCI bridge to [bus 01] [ 2.207757] pci 0000:00:03.1: bridge window [mem 0xe2000000-0xe3ffffff 64bit pref] [ 2.207846] pci 0000:02:00.0: [1022:145a] type 00 class 0x130000 [ 2.207943] pci 0000:02:00.2: [1022:1456] type 00 class 0x108000 [ 2.207960] pci 0000:02:00.2: reg 0x18: [mem 0xf7300000-0xf73fffff] [ 2.207972] pci 0000:02:00.2: reg 0x24: [mem 0xf7400000-0xf7401fff] [ 2.208051] pci 0000:02:00.3: [1022:145f] type 00 class 0x0c0330 [ 2.208063] pci 0000:02:00.3: reg 0x10: [mem 0xf7200000-0xf72fffff 64bit] [ 2.208111] pci 0000:02:00.3: PME# supported from D0 D3hot D3cold [ 2.208168] pci 0000:00:07.1: PCI bridge to [bus 02] [ 2.213137] pci 0000:00:07.1: bridge window [mem 0xf7200000-0xf74fffff] [ 2.213708] pci 0000:03:00.0: [1022:1455] type 00 class 0x130000 [ 2.213816] pci 0000:03:00.1: [1022:1468] type 00 class 0x108000 [ 2.213834] pci 0000:03:00.1: reg 0x18: [mem 0xf7000000-0xf70fffff] [ 2.213846] pci 0000:03:00.1: reg 0x24: [mem 0xf7100000-0xf7101fff] [ 2.213937] pci 0000:00:08.1: PCI bridge to [bus 03] [ 2.218910] pci 0000:00:08.1: bridge window [mem 0xf7000000-0xf71fffff] [ 2.218926] pci_bus 0000:00: on NUMA node 0 [ 2.219300] ACPI: PCI Root Bridge [PC01] (domain 0000 [bus 40-7f]) [ 2.225478] acpi PNP0A08:01: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI] [ 2.233688] acpi PNP0A08:01: PCIe AER handled by firmware [ 2.239132] acpi PNP0A08:01: _OSC: platform does not support [SHPCHotplug] [ 2.246080] acpi PNP0A08:01: _OSC: OS now controls [PCIeHotplug PME PCIeCapability] [ 2.253731] acpi PNP0A08:01: FADT indicates ASPM is unsupported, using BIOS configuration [ 2.262188] PCI host bridge to bus 0000:40 [ 2.266290] pci_bus 0000:40: root bus resource [io 0x4000-0x7fff window] [ 2.273075] pci_bus 0000:40: root bus resource [mem 0xc6000000-0xe0ffffff window] [ 2.280557] pci_bus 0000:40: root bus resource [mem 0x2bf40000000-0x47e7fffffff window] [ 2.288556] pci_bus 0000:40: root bus resource [bus 40-7f] [ 2.294044] pci 0000:40:00.0: [1022:1450] type 00 class 0x060000 [ 2.294116] pci 0000:40:00.2: [1022:1451] type 00 class 0x080600 [ 2.294204] pci 0000:40:01.0: [1022:1452] type 00 class 0x060000 [ 2.294280] pci 0000:40:02.0: [1022:1452] type 00 class 0x060000 [ 2.294354] pci 0000:40:03.0: [1022:1452] type 00 class 0x060000 [ 2.294428] pci 0000:40:04.0: [1022:1452] type 00 class 0x060000 [ 2.294506] pci 0000:40:07.0: [1022:1452] type 00 class 0x060000 [ 2.294569] pci 0000:40:07.1: [1022:1454] type 01 class 0x060400 [ 2.295117] pci 0000:40:07.1: PME# supported from D0 D3hot D3cold [ 2.295196] pci 0000:40:08.0: [1022:1452] type 00 class 0x060000 [ 2.295260] pci 0000:40:08.1: [1022:1454] type 01 class 0x060400 [ 2.295371] pci 0000:40:08.1: PME# supported from D0 D3hot D3cold [ 2.296072] pci 0000:41:00.0: [1022:145a] type 00 class 0x130000 [ 2.296177] pci 0000:41:00.2: [1022:1456] type 00 class 0x108000 [ 2.296196] pci 0000:41:00.2: reg 0x18: [mem 0xdb300000-0xdb3fffff] [ 2.296209] pci 0000:41:00.2: reg 0x24: [mem 0xdb400000-0xdb401fff] [ 2.296295] pci 0000:41:00.3: [1022:145f] type 00 class 0x0c0330 [ 2.296308] pci 0000:41:00.3: reg 0x10: [mem 0xdb200000-0xdb2fffff 64bit] [ 2.296362] pci 0000:41:00.3: PME# supported from D0 D3hot D3cold [ 2.296422] pci 0000:40:07.1: PCI bridge to [bus 41] [ 2.301395] pci 0000:40:07.1: bridge window [mem 0xdb200000-0xdb4fffff] [ 2.301492] pci 0000:42:00.0: [1022:1455] type 00 class 0x130000 [ 2.301610] pci 0000:42:00.1: [1022:1468] type 00 class 0x108000 [ 2.301630] pci 0000:42:00.1: reg 0x18: [mem 0xdb000000-0xdb0fffff] [ 2.301644] pci 0000:42:00.1: reg 0x24: [mem 0xdb100000-0xdb101fff] [ 2.301742] pci 0000:40:08.1: PCI bridge to [bus 42] [ 2.306715] pci 0000:40:08.1: bridge window [mem 0xdb000000-0xdb1fffff] [ 2.306728] pci_bus 0000:40: on NUMA node 1 [ 2.306918] ACPI: PCI Root Bridge [PC02] (domain 0000 [bus 80-bf]) [ 2.313103] acpi PNP0A08:02: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI] [ 2.321310] acpi PNP0A08:02: PCIe AER handled by firmware [ 2.326746] acpi PNP0A08:02: _OSC: platform does not support [SHPCHotplug] [ 2.333693] acpi PNP0A08:02: _OSC: OS now controls [PCIeHotplug PME PCIeCapability] [ 2.341346] acpi PNP0A08:02: FADT indicates ASPM is unsupported, using BIOS configuration [ 2.349826] PCI host bridge to bus 0000:80 [ 2.353931] pci_bus 0000:80: root bus resource [io 0x03b0-0x03df window] [ 2.360717] pci_bus 0000:80: root bus resource [mem 0x000a0000-0x000bffff window] [ 2.368195] pci_bus 0000:80: root bus resource [io 0x8000-0xbfff window] [ 2.374982] pci_bus 0000:80: root bus resource [mem 0xab000000-0xc5ffffff window] [ 2.382462] pci_bus 0000:80: root bus resource [mem 0x47e80000000-0x63dbfffffff window] [ 2.390463] pci_bus 0000:80: root bus resource [bus 80-bf] [ 2.395950] pci 0000:80:00.0: [1022:1450] type 00 class 0x060000 [ 2.396021] pci 0000:80:00.2: [1022:1451] type 00 class 0x080600 [ 2.396111] pci 0000:80:01.0: [1022:1452] type 00 class 0x060000 [ 2.396172] pci 0000:80:01.1: [1022:1453] type 01 class 0x060400 [ 2.396745] pci 0000:80:01.1: PME# supported from D0 D3hot D3cold [ 2.396820] pci 0000:80:01.2: [1022:1453] type 01 class 0x060400 [ 2.396948] pci 0000:80:01.2: PME# supported from D0 D3hot D3cold [ 2.397027] pci 0000:80:02.0: [1022:1452] type 00 class 0x060000 [ 2.397103] pci 0000:80:03.0: [1022:1452] type 00 class 0x060000 [ 2.397163] pci 0000:80:03.1: [1022:1453] type 01 class 0x060400 [ 2.397743] pci 0000:80:03.1: PME# supported from D0 D3hot D3cold [ 2.397839] pci 0000:80:04.0: [1022:1452] type 00 class 0x060000 [ 2.397921] pci 0000:80:07.0: [1022:1452] type 00 class 0x060000 [ 2.397982] pci 0000:80:07.1: [1022:1454] type 01 class 0x060400 [ 2.398090] pci 0000:80:07.1: PME# supported from D0 D3hot D3cold [ 2.398167] pci 0000:80:08.0: [1022:1452] type 00 class 0x060000 [ 2.398228] pci 0000:80:08.1: [1022:1454] type 01 class 0x060400 [ 2.398738] pci 0000:80:08.1: PME# supported from D0 D3hot D3cold [ 2.398951] pci 0000:81:00.0: [14e4:165f] type 00 class 0x020000 [ 2.398977] pci 0000:81:00.0: reg 0x10: [mem 0xac230000-0xac23ffff 64bit pref] [ 2.398991] pci 0000:81:00.0: reg 0x18: [mem 0xac240000-0xac24ffff 64bit pref] [ 2.399006] pci 0000:81:00.0: reg 0x20: [mem 0xac250000-0xac25ffff 64bit pref] [ 2.399016] pci 0000:81:00.0: reg 0x30: [mem 0xfffc0000-0xffffffff pref] [ 2.399092] pci 0000:81:00.0: PME# supported from D0 D3hot D3cold [ 2.399187] pci 0000:81:00.1: [14e4:165f] type 00 class 0x020000 [ 2.399213] pci 0000:81:00.1: reg 0x10: [mem 0xac200000-0xac20ffff 64bit pref] [ 2.399227] pci 0000:81:00.1: reg 0x18: [mem 0xac210000-0xac21ffff 64bit pref] [ 2.399243] pci 0000:81:00.1: reg 0x20: [mem 0xac220000-0xac22ffff 64bit pref] [ 2.399253] pci 0000:81:00.1: reg 0x30: [mem 0xfffc0000-0xffffffff pref] [ 2.399329] pci 0000:81:00.1: PME# supported from D0 D3hot D3cold [ 2.399420] pci 0000:80:01.1: PCI bridge to [bus 81] [ 2.404395] pci 0000:80:01.1: bridge window [mem 0xac200000-0xac2fffff 64bit pref] [ 2.404730] pci 0000:82:00.0: [1556:be00] type 01 class 0x060400 [ 2.407553] pci 0000:80:01.2: PCI bridge to [bus 82-83] [ 2.412785] pci 0000:80:01.2: bridge window [mem 0xc0000000-0xc08fffff] [ 2.412789] pci 0000:80:01.2: bridge window [mem 0xab000000-0xabffffff 64bit pref] [ 2.412837] pci 0000:83:00.0: [102b:0536] type 00 class 0x030000 [ 2.412856] pci 0000:83:00.0: reg 0x10: [mem 0xab000000-0xabffffff pref] [ 2.412867] pci 0000:83:00.0: reg 0x14: [mem 0xc0808000-0xc080bfff] [ 2.412878] pci 0000:83:00.0: reg 0x18: [mem 0xc0000000-0xc07fffff] [ 2.413017] pci 0000:82:00.0: PCI bridge to [bus 83] [ 2.417996] pci 0000:82:00.0: bridge window [mem 0xc0000000-0xc08fffff] [ 2.418002] pci 0000:82:00.0: bridge window [mem 0xab000000-0xabffffff 64bit pref] [ 2.418083] pci 0000:84:00.0: [1000:00d1] type 00 class 0x010700 [ 2.418106] pci 0000:84:00.0: reg 0x10: [mem 0xac000000-0xac0fffff 64bit pref] [ 2.418116] pci 0000:84:00.0: reg 0x18: [mem 0xac100000-0xac1fffff 64bit pref] [ 2.418123] pci 0000:84:00.0: reg 0x20: [mem 0xc0d00000-0xc0dfffff] [ 2.418131] pci 0000:84:00.0: reg 0x24: [io 0x8000-0x80ff] [ 2.418140] pci 0000:84:00.0: reg 0x30: [mem 0x00000000-0x0003ffff pref] [ 2.418191] pci 0000:84:00.0: supports D1 D2 [ 2.420551] pci 0000:80:03.1: PCI bridge to [bus 84] [ 2.425523] pci 0000:80:03.1: bridge window [io 0x8000-0x8fff] [ 2.425525] pci 0000:80:03.1: bridge window [mem 0xc0d00000-0xc0dfffff] [ 2.425529] pci 0000:80:03.1: bridge window [mem 0xac000000-0xac1fffff 64bit pref] [ 2.425768] pci 0000:85:00.0: [1022:145a] type 00 class 0x130000 [ 2.425875] pci 0000:85:00.2: [1022:1456] type 00 class 0x108000 [ 2.425894] pci 0000:85:00.2: reg 0x18: [mem 0xc0b00000-0xc0bfffff] [ 2.425907] pci 0000:85:00.2: reg 0x24: [mem 0xc0c00000-0xc0c01fff] [ 2.425998] pci 0000:80:07.1: PCI bridge to [bus 85] [ 2.430965] pci 0000:80:07.1: bridge window [mem 0xc0b00000-0xc0cfffff] [ 2.431059] pci 0000:86:00.0: [1022:1455] type 00 class 0x130000 [ 2.431176] pci 0000:86:00.1: [1022:1468] type 00 class 0x108000 [ 2.431196] pci 0000:86:00.1: reg 0x18: [mem 0xc0900000-0xc09fffff] [ 2.431210] pci 0000:86:00.1: reg 0x24: [mem 0xc0a00000-0xc0a01fff] [ 2.431301] pci 0000:86:00.2: [1022:7901] type 00 class 0x010601 [ 2.431333] pci 0000:86:00.2: reg 0x24: [mem 0xc0a02000-0xc0a02fff] [ 2.431372] pci 0000:86:00.2: PME# supported from D3hot D3cold [ 2.431436] pci 0000:80:08.1: PCI bridge to [bus 86] [ 2.436407] pci 0000:80:08.1: bridge window [mem 0xc0900000-0xc0afffff] [ 2.436433] pci_bus 0000:80: on NUMA node 2 [ 2.436605] ACPI: PCI Root Bridge [PC03] (domain 0000 [bus c0-ff]) [ 2.442786] acpi PNP0A08:03: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI] [ 2.450994] acpi PNP0A08:03: PCIe AER handled by firmware [ 2.456430] acpi PNP0A08:03: _OSC: platform does not support [SHPCHotplug] [ 2.463378] acpi PNP0A08:03: _OSC: OS now controls [PCIeHotplug PME PCIeCapability] [ 2.471030] acpi PNP0A08:03: FADT indicates ASPM is unsupported, using BIOS configuration [ 2.479399] acpi PNP0A08:03: host bridge window [mem 0x63dc0000000-0xffffffffffff window] ([0x80000000000-0xffffffffffff] ignored, not CPU addressable) [ 2.493035] PCI host bridge to bus 0000:c0 [ 2.497135] pci_bus 0000:c0: root bus resource [io 0xc000-0xffff window] [ 2.503919] pci_bus 0000:c0: root bus resource [mem 0x90000000-0xaaffffff window] [ 2.511400] pci_bus 0000:c0: root bus resource [mem 0x63dc0000000-0x7ffffffffff window] [ 2.519398] pci_bus 0000:c0: root bus resource [bus c0-ff] [ 2.524889] pci 0000:c0:00.0: [1022:1450] type 00 class 0x060000 [ 2.524957] pci 0000:c0:00.2: [1022:1451] type 00 class 0x080600 [ 2.525048] pci 0000:c0:01.0: [1022:1452] type 00 class 0x060000 [ 2.525108] pci 0000:c0:01.1: [1022:1453] type 01 class 0x060400 [ 2.525227] pci 0000:c0:01.1: PME# supported from D0 D3hot D3cold [ 2.525326] pci 0000:c0:02.0: [1022:1452] type 00 class 0x060000 [ 2.525402] pci 0000:c0:03.0: [1022:1452] type 00 class 0x060000 [ 2.525476] pci 0000:c0:04.0: [1022:1452] type 00 class 0x060000 [ 2.525558] pci 0000:c0:07.0: [1022:1452] type 00 class 0x060000 [ 2.525618] pci 0000:c0:07.1: [1022:1454] type 01 class 0x060400 [ 2.526031] pci 0000:c0:07.1: PME# supported from D0 D3hot D3cold [ 2.526110] pci 0000:c0:08.0: [1022:1452] type 00 class 0x060000 [ 2.526172] pci 0000:c0:08.1: [1022:1454] type 01 class 0x060400 [ 2.526285] pci 0000:c0:08.1: PME# supported from D0 D3hot D3cold [ 2.526483] pci 0000:c1:00.0: [1000:005f] type 00 class 0x010400 [ 2.526497] pci 0000:c1:00.0: reg 0x10: [io 0xc000-0xc0ff] [ 2.526507] pci 0000:c1:00.0: reg 0x14: [mem 0xa5500000-0xa550ffff 64bit] [ 2.526517] pci 0000:c1:00.0: reg 0x1c: [mem 0xa5400000-0xa54fffff 64bit] [ 2.526529] pci 0000:c1:00.0: reg 0x30: [mem 0xfff00000-0xffffffff pref] [ 2.526579] pci 0000:c1:00.0: supports D1 D2 [ 2.526632] pci 0000:c0:01.1: PCI bridge to [bus c1] [ 2.531605] pci 0000:c0:01.1: bridge window [io 0xc000-0xcfff] [ 2.531607] pci 0000:c0:01.1: bridge window [mem 0xa5400000-0xa55fffff] [ 2.532022] pci 0000:c2:00.0: [1022:145a] type 00 class 0x130000 [ 2.532127] pci 0000:c2:00.2: [1022:1456] type 00 class 0x108000 [ 2.532146] pci 0000:c2:00.2: reg 0x18: [mem 0xa5200000-0xa52fffff] [ 2.532159] pci 0000:c2:00.2: reg 0x24: [mem 0xa5300000-0xa5301fff] [ 2.532252] pci 0000:c0:07.1: PCI bridge to [bus c2] [ 2.537222] pci 0000:c0:07.1: bridge window [mem 0xa5200000-0xa53fffff] [ 2.537317] pci 0000:c3:00.0: [1022:1455] type 00 class 0x130000 [ 2.537435] pci 0000:c3:00.1: [1022:1468] type 00 class 0x108000 [ 2.537454] pci 0000:c3:00.1: reg 0x18: [mem 0xa5000000-0xa50fffff] [ 2.537468] pci 0000:c3:00.1: reg 0x24: [mem 0xa5100000-0xa5101fff] [ 2.537567] pci 0000:c0:08.1: PCI bridge to [bus c3] [ 2.542534] pci 0000:c0:08.1: bridge window [mem 0xa5000000-0xa51fffff] [ 2.542550] pci_bus 0000:c0: on NUMA node 3 [ 2.544667] vgaarb: device added: PCI:0000:83:00.0,decodes=io+mem,owns=io+mem,locks=none [ 2.552753] vgaarb: loaded [ 2.555460] vgaarb: bridge control possible 0000:83:00.0 [ 2.560884] SCSI subsystem initialized [ 2.564666] ACPI: bus type USB registered [ 2.568696] usbcore: registered new interface driver usbfs [ 2.574190] usbcore: registered new interface driver hub [ 2.579717] usbcore: registered new device driver usb [ 2.585093] EDAC MC: Ver: 3.0.0 [ 2.588486] PCI: Using ACPI for IRQ routing [ 2.611640] PCI: pci_cache_line_size set to 64 bytes [ 2.611782] e820: reserve RAM buffer [mem 0x0008f000-0x0008ffff] [ 2.611784] e820: reserve RAM buffer [mem 0x37833020-0x37ffffff] [ 2.611786] e820: reserve RAM buffer [mem 0x3784c020-0x37ffffff] [ 2.611788] e820: reserve RAM buffer [mem 0x37872020-0x37ffffff] [ 2.611790] e820: reserve RAM buffer [mem 0x3787b020-0x37ffffff] [ 2.611791] e820: reserve RAM buffer [mem 0x378ad020-0x37ffffff] [ 2.611792] e820: reserve RAM buffer [mem 0x378df020-0x37ffffff] [ 2.611794] e820: reserve RAM buffer [mem 0x4f723000-0x4fffffff] [ 2.611795] e820: reserve RAM buffer [mem 0x6cacf000-0x6fffffff] [ 2.611796] e820: reserve RAM buffer [mem 0x107f380000-0x107fffffff] [ 2.611798] e820: reserve RAM buffer [mem 0x207ff80000-0x207fffffff] [ 2.611799] e820: reserve RAM buffer [mem 0x307ff80000-0x307fffffff] [ 2.611800] e820: reserve RAM buffer [mem 0x407ff80000-0x407fffffff] [ 2.612039] NetLabel: Initializing [ 2.615446] NetLabel: domain hash size = 128 [ 2.619807] NetLabel: protocols = UNLABELED CIPSOv4 [ 2.624788] NetLabel: unlabeled traffic allowed by default [ 2.630558] hpet0: at MMIO 0xfed00000, IRQs 2, 8, 0 [ 2.635545] hpet0: 3 comparators, 32-bit 14.318180 MHz counter [ 2.643551] Switched to clocksource hpet [ 2.652124] pnp: PnP ACPI init [ 2.655200] ACPI: bus type PNP registered [ 2.659388] system 00:00: [mem 0x80000000-0x8fffffff] has been reserved [ 2.666012] system 00:00: Plug and Play ACPI device, IDs PNP0c01 (active) [ 2.666066] pnp 00:01: Plug and Play ACPI device, IDs PNP0b00 (active) [ 2.666257] pnp 00:02: Plug and Play ACPI device, IDs PNP0501 (active) [ 2.666435] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active) [ 2.666610] pnp: PnP ACPI: found 4 devices [ 2.670714] ACPI: bus type PNP unregistered [ 2.682145] pci 0000:01:00.0: can't claim BAR 6 [mem 0xfff00000-0xffffffff pref]: no compatible bridge window [ 2.692064] pci 0000:81:00.0: can't claim BAR 6 [mem 0xfffc0000-0xffffffff pref]: no compatible bridge window [ 2.701977] pci 0000:81:00.1: can't claim BAR 6 [mem 0xfffc0000-0xffffffff pref]: no compatible bridge window [ 2.711894] pci 0000:c1:00.0: can't claim BAR 6 [mem 0xfff00000-0xffffffff pref]: no compatible bridge window [ 2.721829] pci 0000:00:03.1: BAR 14: assigned [mem 0xe1000000-0xe10fffff] [ 2.728709] pci 0000:01:00.0: BAR 6: assigned [mem 0xe1000000-0xe10fffff pref] [ 2.735936] pci 0000:00:03.1: PCI bridge to [bus 01] [ 2.740913] pci 0000:00:03.1: bridge window [mem 0xe1000000-0xe10fffff] [ 2.747705] pci 0000:00:03.1: bridge window [mem 0xe2000000-0xe3ffffff 64bit pref] [ 2.755457] pci 0000:00:07.1: PCI bridge to [bus 02] [ 2.760436] pci 0000:00:07.1: bridge window [mem 0xf7200000-0xf74fffff] [ 2.767236] pci 0000:00:08.1: PCI bridge to [bus 03] [ 2.772217] pci 0000:00:08.1: bridge window [mem 0xf7000000-0xf71fffff] [ 2.779014] pci_bus 0000:00: resource 4 [io 0x0000-0x03af window] [ 2.779016] pci_bus 0000:00: resource 5 [io 0x03e0-0x0cf7 window] [ 2.779018] pci_bus 0000:00: resource 6 [mem 0x000c0000-0x000c3fff window] [ 2.779020] pci_bus 0000:00: resource 7 [mem 0x000c4000-0x000c7fff window] [ 2.779022] pci_bus 0000:00: resource 8 [mem 0x000c8000-0x000cbfff window] [ 2.779023] pci_bus 0000:00: resource 9 [mem 0x000cc000-0x000cffff window] [ 2.779025] pci_bus 0000:00: resource 10 [mem 0x000d0000-0x000d3fff window] [ 2.779027] pci_bus 0000:00: resource 11 [mem 0x000d4000-0x000d7fff window] [ 2.779028] pci_bus 0000:00: resource 12 [mem 0x000d8000-0x000dbfff window] [ 2.779030] pci_bus 0000:00: resource 13 [mem 0x000dc000-0x000dffff window] [ 2.779032] pci_bus 0000:00: resource 14 [mem 0x000e0000-0x000e3fff window] [ 2.779034] pci_bus 0000:00: resource 15 [mem 0x000e4000-0x000e7fff window] [ 2.779035] pci_bus 0000:00: resource 16 [mem 0x000e8000-0x000ebfff window] [ 2.779037] pci_bus 0000:00: resource 17 [mem 0x000ec000-0x000effff window] [ 2.779039] pci_bus 0000:00: resource 18 [mem 0x000f0000-0x000fffff window] [ 2.779040] pci_bus 0000:00: resource 19 [io 0x0d00-0x3fff window] [ 2.779042] pci_bus 0000:00: resource 20 [mem 0xe1000000-0xfebfffff window] [ 2.779044] pci_bus 0000:00: resource 21 [mem 0x10000000000-0x2bf3fffffff window] [ 2.779046] pci_bus 0000:01: resource 1 [mem 0xe1000000-0xe10fffff] [ 2.779047] pci_bus 0000:01: resource 2 [mem 0xe2000000-0xe3ffffff 64bit pref] [ 2.779049] pci_bus 0000:02: resource 1 [mem 0xf7200000-0xf74fffff] [ 2.779051] pci_bus 0000:03: resource 1 [mem 0xf7000000-0xf71fffff] [ 2.779062] pci 0000:40:07.1: PCI bridge to [bus 41] [ 2.784037] pci 0000:40:07.1: bridge window [mem 0xdb200000-0xdb4fffff] [ 2.790836] pci 0000:40:08.1: PCI bridge to [bus 42] [ 2.795816] pci 0000:40:08.1: bridge window [mem 0xdb000000-0xdb1fffff] [ 2.802613] pci_bus 0000:40: resource 4 [io 0x4000-0x7fff window] [ 2.802614] pci_bus 0000:40: resource 5 [mem 0xc6000000-0xe0ffffff window] [ 2.802616] pci_bus 0000:40: resource 6 [mem 0x2bf40000000-0x47e7fffffff window] [ 2.802618] pci_bus 0000:41: resource 1 [mem 0xdb200000-0xdb4fffff] [ 2.802620] pci_bus 0000:42: resource 1 [mem 0xdb000000-0xdb1fffff] [ 2.802651] pci 0000:80:01.1: BAR 14: assigned [mem 0xac300000-0xac3fffff] [ 2.809544] pci 0000:81:00.0: BAR 6: assigned [mem 0xac300000-0xac33ffff pref] [ 2.816773] pci 0000:81:00.1: BAR 6: assigned [mem 0xac340000-0xac37ffff pref] [ 2.823999] pci 0000:80:01.1: PCI bridge to [bus 81] [ 2.828976] pci 0000:80:01.1: bridge window [mem 0xac300000-0xac3fffff] [ 2.835770] pci 0000:80:01.1: bridge window [mem 0xac200000-0xac2fffff 64bit pref] [ 2.843520] pci 0000:82:00.0: PCI bridge to [bus 83] [ 2.848496] pci 0000:82:00.0: bridge window [mem 0xc0000000-0xc08fffff] [ 2.855291] pci 0000:82:00.0: bridge window [mem 0xab000000-0xabffffff 64bit pref] [ 2.863041] pci 0000:80:01.2: PCI bridge to [bus 82-83] [ 2.868282] pci 0000:80:01.2: bridge window [mem 0xc0000000-0xc08fffff] [ 2.875075] pci 0000:80:01.2: bridge window [mem 0xab000000-0xabffffff 64bit pref] [ 2.882827] pci 0000:84:00.0: BAR 6: no space for [mem size 0x00040000 pref] [ 2.889879] pci 0000:84:00.0: BAR 6: failed to assign [mem size 0x00040000 pref] [ 2.897279] pci 0000:80:03.1: PCI bridge to [bus 84] [ 2.902253] pci 0000:80:03.1: bridge window [io 0x8000-0x8fff] [ 2.908356] pci 0000:80:03.1: bridge window [mem 0xc0d00000-0xc0dfffff] [ 2.915151] pci 0000:80:03.1: bridge window [mem 0xac000000-0xac1fffff 64bit pref] [ 2.922899] pci 0000:80:07.1: PCI bridge to [bus 85] [ 2.927874] pci 0000:80:07.1: bridge window [mem 0xc0b00000-0xc0cfffff] [ 2.934671] pci 0000:80:08.1: PCI bridge to [bus 86] [ 2.939646] pci 0000:80:08.1: bridge window [mem 0xc0900000-0xc0afffff] [ 2.946442] pci_bus 0000:80: resource 4 [io 0x03b0-0x03df window] [ 2.946444] pci_bus 0000:80: resource 5 [mem 0x000a0000-0x000bffff window] [ 2.946446] pci_bus 0000:80: resource 6 [io 0x8000-0xbfff window] [ 2.946448] pci_bus 0000:80: resource 7 [mem 0xab000000-0xc5ffffff window] [ 2.946449] pci_bus 0000:80: resource 8 [mem 0x47e80000000-0x63dbfffffff window] [ 2.946451] pci_bus 0000:81: resource 1 [mem 0xac300000-0xac3fffff] [ 2.946453] pci_bus 0000:81: resource 2 [mem 0xac200000-0xac2fffff 64bit pref] [ 2.946455] pci_bus 0000:82: resource 1 [mem 0xc0000000-0xc08fffff] [ 2.946456] pci_bus 0000:82: resource 2 [mem 0xab000000-0xabffffff 64bit pref] [ 2.946458] pci_bus 0000:83: resource 1 [mem 0xc0000000-0xc08fffff] [ 2.946460] pci_bus 0000:83: resource 2 [mem 0xab000000-0xabffffff 64bit pref] [ 2.946461] pci_bus 0000:84: resource 0 [io 0x8000-0x8fff] [ 2.946463] pci_bus 0000:84: resource 1 [mem 0xc0d00000-0xc0dfffff] [ 2.946465] pci_bus 0000:84: resource 2 [mem 0xac000000-0xac1fffff 64bit pref] [ 2.946466] pci_bus 0000:85: resource 1 [mem 0xc0b00000-0xc0cfffff] [ 2.946468] pci_bus 0000:86: resource 1 [mem 0xc0900000-0xc0afffff] [ 2.946483] pci 0000:c1:00.0: BAR 6: no space for [mem size 0x00100000 pref] [ 2.953547] pci 0000:c1:00.0: BAR 6: failed to assign [mem size 0x00100000 pref] [ 2.960945] pci 0000:c0:01.1: PCI bridge to [bus c1] [ 2.965922] pci 0000:c0:01.1: bridge window [io 0xc000-0xcfff] [ 2.972025] pci 0000:c0:01.1: bridge window [mem 0xa5400000-0xa55fffff] [ 2.978822] pci 0000:c0:07.1: PCI bridge to [bus c2] [ 2.983802] pci 0000:c0:07.1: bridge window [mem 0xa5200000-0xa53fffff] [ 2.990599] pci 0000:c0:08.1: PCI bridge to [bus c3] [ 2.995573] pci 0000:c0:08.1: bridge window [mem 0xa5000000-0xa51fffff] [ 3.002369] pci_bus 0000:c0: resource 4 [io 0xc000-0xffff window] [ 3.002371] pci_bus 0000:c0: resource 5 [mem 0x90000000-0xaaffffff window] [ 3.002372] pci_bus 0000:c0: resource 6 [mem 0x63dc0000000-0x7ffffffffff window] [ 3.002374] pci_bus 0000:c1: resource 0 [io 0xc000-0xcfff] [ 3.002376] pci_bus 0000:c1: resource 1 [mem 0xa5400000-0xa55fffff] [ 3.002378] pci_bus 0000:c2: resource 1 [mem 0xa5200000-0xa53fffff] [ 3.002379] pci_bus 0000:c3: resource 1 [mem 0xa5000000-0xa51fffff] [ 3.002461] NET: Registered protocol family 2 [ 3.007489] TCP established hash table entries: 524288 (order: 10, 4194304 bytes) [ 3.016163] TCP bind hash table entries: 65536 (order: 8, 1048576 bytes) [ 3.022985] TCP: Hash tables configured (established 524288 bind 65536) [ 3.029626] TCP: reno registered [ 3.032961] UDP hash table entries: 65536 (order: 9, 2097152 bytes) [ 3.039579] UDP-Lite hash table entries: 65536 (order: 9, 2097152 bytes) [ 3.046777] NET: Registered protocol family 1 [ 3.051573] pci 0000:83:00.0: Boot video device [ 3.051608] PCI: CLS 64 bytes, default 64 [ 3.051648] Unpacking initramfs... [ 3.320988] Freeing initrd memory: 19600k freed [ 3.327655] AMD-Vi: IOMMU performance counters supported [ 3.333035] AMD-Vi: IOMMU performance counters supported [ 3.338388] AMD-Vi: IOMMU performance counters supported [ 3.343739] AMD-Vi: IOMMU performance counters supported [ 3.350376] iommu: Adding device 0000:00:01.0 to group 0 [ 3.356412] iommu: Adding device 0000:00:02.0 to group 1 [ 3.362442] iommu: Adding device 0000:00:03.0 to group 2 [ 3.368588] iommu: Adding device 0000:00:03.1 to group 3 [ 3.374574] iommu: Adding device 0000:00:04.0 to group 4 [ 3.380587] iommu: Adding device 0000:00:07.0 to group 5 [ 3.386592] iommu: Adding device 0000:00:07.1 to group 6 [ 3.392624] iommu: Adding device 0000:00:08.0 to group 7 [ 3.398623] iommu: Adding device 0000:00:08.1 to group 8 [ 3.404689] iommu: Adding device 0000:00:14.0 to group 9 [ 3.410024] iommu: Adding device 0000:00:14.3 to group 9 [ 3.416101] iommu: Adding device 0000:00:18.0 to group 10 [ 3.421531] iommu: Adding device 0000:00:18.1 to group 10 [ 3.426960] iommu: Adding device 0000:00:18.2 to group 10 [ 3.432383] iommu: Adding device 0000:00:18.3 to group 10 [ 3.437807] iommu: Adding device 0000:00:18.4 to group 10 [ 3.443237] iommu: Adding device 0000:00:18.5 to group 10 [ 3.448663] iommu: Adding device 0000:00:18.6 to group 10 [ 3.454091] iommu: Adding device 0000:00:18.7 to group 10 [ 3.460274] iommu: Adding device 0000:00:19.0 to group 11 [ 3.465702] iommu: Adding device 0000:00:19.1 to group 11 [ 3.471125] iommu: Adding device 0000:00:19.2 to group 11 [ 3.476562] iommu: Adding device 0000:00:19.3 to group 11 [ 3.481986] iommu: Adding device 0000:00:19.4 to group 11 [ 3.487413] iommu: Adding device 0000:00:19.5 to group 11 [ 3.492841] iommu: Adding device 0000:00:19.6 to group 11 [ 3.498261] iommu: Adding device 0000:00:19.7 to group 11 [ 3.504401] iommu: Adding device 0000:00:1a.0 to group 12 [ 3.509828] iommu: Adding device 0000:00:1a.1 to group 12 [ 3.515251] iommu: Adding device 0000:00:1a.2 to group 12 [ 3.520678] iommu: Adding device 0000:00:1a.3 to group 12 [ 3.526105] iommu: Adding device 0000:00:1a.4 to group 12 [ 3.531526] iommu: Adding device 0000:00:1a.5 to group 12 [ 3.536951] iommu: Adding device 0000:00:1a.6 to group 12 [ 3.542379] iommu: Adding device 0000:00:1a.7 to group 12 [ 3.548551] iommu: Adding device 0000:00:1b.0 to group 13 [ 3.553986] iommu: Adding device 0000:00:1b.1 to group 13 [ 3.559411] iommu: Adding device 0000:00:1b.2 to group 13 [ 3.564837] iommu: Adding device 0000:00:1b.3 to group 13 [ 3.570259] iommu: Adding device 0000:00:1b.4 to group 13 [ 3.575686] iommu: Adding device 0000:00:1b.5 to group 13 [ 3.581113] iommu: Adding device 0000:00:1b.6 to group 13 [ 3.586540] iommu: Adding device 0000:00:1b.7 to group 13 [ 3.592674] iommu: Adding device 0000:01:00.0 to group 14 [ 3.598788] iommu: Adding device 0000:02:00.0 to group 15 [ 3.604876] iommu: Adding device 0000:02:00.2 to group 16 [ 3.610971] iommu: Adding device 0000:02:00.3 to group 17 [ 3.617049] iommu: Adding device 0000:03:00.0 to group 18 [ 3.623164] iommu: Adding device 0000:03:00.1 to group 19 [ 3.629275] iommu: Adding device 0000:40:01.0 to group 20 [ 3.635380] iommu: Adding device 0000:40:02.0 to group 21 [ 3.641463] iommu: Adding device 0000:40:03.0 to group 22 [ 3.647532] iommu: Adding device 0000:40:04.0 to group 23 [ 3.653599] iommu: Adding device 0000:40:07.0 to group 24 [ 3.659634] iommu: Adding device 0000:40:07.1 to group 25 [ 3.665680] iommu: Adding device 0000:40:08.0 to group 26 [ 3.671724] iommu: Adding device 0000:40:08.1 to group 27 [ 3.677760] iommu: Adding device 0000:41:00.0 to group 28 [ 3.683825] iommu: Adding device 0000:41:00.2 to group 29 [ 3.689857] iommu: Adding device 0000:41:00.3 to group 30 [ 3.695890] iommu: Adding device 0000:42:00.0 to group 31 [ 3.701919] iommu: Adding device 0000:42:00.1 to group 32 [ 3.707954] iommu: Adding device 0000:80:01.0 to group 33 [ 3.713961] iommu: Adding device 0000:80:01.1 to group 34 [ 3.720137] iommu: Adding device 0000:80:01.2 to group 35 [ 3.726185] iommu: Adding device 0000:80:02.0 to group 36 [ 3.732258] iommu: Adding device 0000:80:03.0 to group 37 [ 3.738252] iommu: Adding device 0000:80:03.1 to group 38 [ 3.744332] iommu: Adding device 0000:80:04.0 to group 39 [ 3.750381] iommu: Adding device 0000:80:07.0 to group 40 [ 3.756393] iommu: Adding device 0000:80:07.1 to group 41 [ 3.762458] iommu: Adding device 0000:80:08.0 to group 42 [ 3.768515] iommu: Adding device 0000:80:08.1 to group 43 [ 3.774593] iommu: Adding device 0000:81:00.0 to group 44 [ 3.780038] iommu: Adding device 0000:81:00.1 to group 44 [ 3.786072] iommu: Adding device 0000:82:00.0 to group 45 [ 3.791492] iommu: Adding device 0000:83:00.0 to group 45 [ 3.797537] iommu: Adding device 0000:84:00.0 to group 46 [ 3.803538] iommu: Adding device 0000:85:00.0 to group 47 [ 3.809578] iommu: Adding device 0000:85:00.2 to group 48 [ 3.815649] iommu: Adding device 0000:86:00.0 to group 49 [ 3.821671] iommu: Adding device 0000:86:00.1 to group 50 [ 3.827741] iommu: Adding device 0000:86:00.2 to group 51 [ 3.833811] iommu: Adding device 0000:c0:01.0 to group 52 [ 3.839814] iommu: Adding device 0000:c0:01.1 to group 53 [ 3.845838] iommu: Adding device 0000:c0:02.0 to group 54 [ 3.851865] iommu: Adding device 0000:c0:03.0 to group 55 [ 3.857881] iommu: Adding device 0000:c0:04.0 to group 56 [ 3.863927] iommu: Adding device 0000:c0:07.0 to group 57 [ 3.869965] iommu: Adding device 0000:c0:07.1 to group 58 [ 3.876019] iommu: Adding device 0000:c0:08.0 to group 59 [ 3.882093] iommu: Adding device 0000:c0:08.1 to group 60 [ 3.890512] iommu: Adding device 0000:c1:00.0 to group 61 [ 3.896571] iommu: Adding device 0000:c2:00.0 to group 62 [ 3.902651] iommu: Adding device 0000:c2:00.2 to group 63 [ 3.908671] iommu: Adding device 0000:c3:00.0 to group 64 [ 3.914713] iommu: Adding device 0000:c3:00.1 to group 65 [ 3.920317] AMD-Vi: Found IOMMU at 0000:00:00.2 cap 0x40 [ 3.925639] AMD-Vi: Extended features (0xf77ef22294ada): [ 3.930961] PPR NX GT IA GA PC GA_vAPIC [ 3.935095] AMD-Vi: Found IOMMU at 0000:40:00.2 cap 0x40 [ 3.940416] AMD-Vi: Extended features (0xf77ef22294ada): [ 3.945739] PPR NX GT IA GA PC GA_vAPIC [ 3.949879] AMD-Vi: Found IOMMU at 0000:80:00.2 cap 0x40 [ 3.955202] AMD-Vi: Extended features (0xf77ef22294ada): [ 3.960522] PPR NX GT IA GA PC GA_vAPIC [ 3.964666] AMD-Vi: Found IOMMU at 0000:c0:00.2 cap 0x40 [ 3.969987] AMD-Vi: Extended features (0xf77ef22294ada): [ 3.975310] PPR NX GT IA GA PC GA_vAPIC [ 3.979444] AMD-Vi: Interrupt remapping enabled [ 3.983984] AMD-Vi: virtual APIC enabled [ 3.987980] pci 0000:00:00.2: irq 26 for MSI/MSI-X [ 3.988070] pci 0000:40:00.2: irq 27 for MSI/MSI-X [ 3.988154] pci 0000:80:00.2: irq 28 for MSI/MSI-X [ 3.988235] pci 0000:c0:00.2: irq 29 for MSI/MSI-X [ 3.988290] AMD-Vi: Lazy IO/TLB flushing enabled [ 3.994620] perf: AMD NB counters detected [ 3.998769] perf: AMD LLC counters detected [ 4.008922] sha1_ssse3: Using SHA-NI optimized SHA-1 implementation [ 4.015272] sha256_ssse3: Using SHA-256-NI optimized SHA-256 implementation [ 4.023854] futex hash table entries: 32768 (order: 9, 2097152 bytes) [ 4.030485] Initialise system trusted keyring [ 4.034886] audit: initializing netlink socket (disabled) [ 4.040302] type=2000 audit(1594225430.195:1): initialized [ 4.071144] HugeTLB registered 1 GB page size, pre-allocated 0 pages [ 4.077502] HugeTLB registered 2 MB page size, pre-allocated 0 pages [ 4.085119] zpool: loaded [ 4.087754] zbud: loaded [ 4.090660] VFS: Disk quotas dquot_6.6.0 [ 4.094692] Dquot-cache hash table entries: 512 (order 0, 4096 bytes) [ 4.101501] msgmni has been set to 32768 [ 4.105529] Key type big_key registered [ 4.109378] SELinux: Registering netfilter hooks [ 4.111798] NET: Registered protocol family 38 [ 4.116256] Key type asymmetric registered [ 4.120360] Asymmetric key parser 'x509' registered [ 4.125293] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 248) [ 4.132837] io scheduler noop registered [ 4.136776] io scheduler deadline registered (default) [ 4.141959] io scheduler cfq registered [ 4.145806] io scheduler mq-deadline registered [ 4.150347] io scheduler kyber registered [ 4.155374] pcieport 0000:00:03.1: irq 30 for MSI/MSI-X [ 4.155546] pcieport 0000:00:07.1: irq 31 for MSI/MSI-X [ 4.156428] pcieport 0000:00:08.1: irq 33 for MSI/MSI-X [ 4.156697] pcieport 0000:40:07.1: irq 34 for MSI/MSI-X [ 4.157005] pcieport 0000:40:08.1: irq 36 for MSI/MSI-X [ 4.157319] pcieport 0000:80:01.1: irq 37 for MSI/MSI-X [ 4.158056] pcieport 0000:80:01.2: irq 38 for MSI/MSI-X [ 4.158264] pcieport 0000:80:03.1: irq 39 for MSI/MSI-X [ 4.159039] pcieport 0000:80:07.1: irq 41 for MSI/MSI-X [ 4.159284] pcieport 0000:80:08.1: irq 43 for MSI/MSI-X [ 4.160088] pcieport 0000:c0:01.1: irq 44 for MSI/MSI-X [ 4.160411] pcieport 0000:c0:07.1: irq 46 for MSI/MSI-X [ 4.161142] pcieport 0000:c0:08.1: irq 48 for MSI/MSI-X [ 4.161255] pcieport 0000:00:03.1: Signaling PME through PCIe PME interrupt [ 4.168219] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt [ 4.174754] pcie_pme 0000:00:03.1:pcie001: service driver pcie_pme loaded [ 4.174765] pcieport 0000:00:07.1: Signaling PME through PCIe PME interrupt [ 4.181730] pci 0000:02:00.0: Signaling PME through PCIe PME interrupt [ 4.188265] pci 0000:02:00.2: Signaling PME through PCIe PME interrupt [ 4.194800] pci 0000:02:00.3: Signaling PME through PCIe PME interrupt [ 4.201335] pcie_pme 0000:00:07.1:pcie001: service driver pcie_pme loaded [ 4.201349] pcieport 0000:00:08.1: Signaling PME through PCIe PME interrupt [ 4.208311] pci 0000:03:00.0: Signaling PME through PCIe PME interrupt [ 4.214846] pci 0000:03:00.1: Signaling PME through PCIe PME interrupt [ 4.221381] pcie_pme 0000:00:08.1:pcie001: service driver pcie_pme loaded [ 4.221399] pcieport 0000:40:07.1: Signaling PME through PCIe PME interrupt [ 4.228366] pci 0000:41:00.0: Signaling PME through PCIe PME interrupt [ 4.234901] pci 0000:41:00.2: Signaling PME through PCIe PME interrupt [ 4.241436] pci 0000:41:00.3: Signaling PME through PCIe PME interrupt [ 4.247971] pcie_pme 0000:40:07.1:pcie001: service driver pcie_pme loaded [ 4.247985] pcieport 0000:40:08.1: Signaling PME through PCIe PME interrupt [ 4.254957] pci 0000:42:00.0: Signaling PME through PCIe PME interrupt [ 4.261491] pci 0000:42:00.1: Signaling PME through PCIe PME interrupt [ 4.268026] pcie_pme 0000:40:08.1:pcie001: service driver pcie_pme loaded [ 4.268042] pcieport 0000:80:01.1: Signaling PME through PCIe PME interrupt [ 4.275012] pci 0000:81:00.0: Signaling PME through PCIe PME interrupt [ 4.281546] pci 0000:81:00.1: Signaling PME through PCIe PME interrupt [ 4.288084] pcie_pme 0000:80:01.1:pcie001: service driver pcie_pme loaded [ 4.288098] pcieport 0000:80:01.2: Signaling PME through PCIe PME interrupt [ 4.295067] pci 0000:82:00.0: Signaling PME through PCIe PME interrupt [ 4.301601] pci 0000:83:00.0: Signaling PME through PCIe PME interrupt [ 4.308139] pcie_pme 0000:80:01.2:pcie001: service driver pcie_pme loaded [ 4.308153] pcieport 0000:80:03.1: Signaling PME through PCIe PME interrupt [ 4.315122] pci 0000:84:00.0: Signaling PME through PCIe PME interrupt [ 4.321658] pcie_pme 0000:80:03.1:pcie001: service driver pcie_pme loaded [ 4.321677] pcieport 0000:80:07.1: Signaling PME through PCIe PME interrupt [ 4.328642] pci 0000:85:00.0: Signaling PME through PCIe PME interrupt [ 4.335178] pci 0000:85:00.2: Signaling PME through PCIe PME interrupt [ 4.341713] pcie_pme 0000:80:07.1:pcie001: service driver pcie_pme loaded [ 4.341728] pcieport 0000:80:08.1: Signaling PME through PCIe PME interrupt [ 4.348700] pci 0000:86:00.0: Signaling PME through PCIe PME interrupt [ 4.355232] pci 0000:86:00.1: Signaling PME through PCIe PME interrupt [ 4.361769] pci 0000:86:00.2: Signaling PME through PCIe PME interrupt [ 4.368305] pcie_pme 0000:80:08.1:pcie001: service driver pcie_pme loaded [ 4.368321] pcieport 0000:c0:01.1: Signaling PME through PCIe PME interrupt [ 4.375288] pci 0000:c1:00.0: Signaling PME through PCIe PME interrupt [ 4.381824] pcie_pme 0000:c0:01.1:pcie001: service driver pcie_pme loaded [ 4.381839] pcieport 0000:c0:07.1: Signaling PME through PCIe PME interrupt [ 4.388810] pci 0000:c2:00.0: Signaling PME through PCIe PME interrupt [ 4.395343] pci 0000:c2:00.2: Signaling PME through PCIe PME interrupt [ 4.401881] pcie_pme 0000:c0:07.1:pcie001: service driver pcie_pme loaded [ 4.401894] pcieport 0000:c0:08.1: Signaling PME through PCIe PME interrupt [ 4.408864] pci 0000:c3:00.0: Signaling PME through PCIe PME interrupt [ 4.415400] pci 0000:c3:00.1: Signaling PME through PCIe PME interrupt [ 4.421937] pcie_pme 0000:c0:08.1:pcie001: service driver pcie_pme loaded [ 4.421956] pci_hotplug: PCI Hot Plug PCI Core version: 0.5 [ 4.427538] pciehp: PCI Express Hot Plug Controller Driver version: 0.4 [ 4.434207] shpchp: Standard Hot Plug PCI Controller Driver version: 0.4 [ 4.441017] efifb: probing for efifb [ 4.444611] efifb: framebuffer at 0xab000000, mapped to 0xffffa93099800000, using 3072k, total 3072k [ 4.453743] efifb: mode is 1024x768x32, linelength=4096, pages=1 [ 4.459758] efifb: scrolling: redraw [ 4.463347] efifb: Truecolor: size=8:8:8:8, shift=24:16:8:0 [ 4.484673] Console: switching to colour frame buffer device 128x48 [ 4.506499] fb0: EFI VGA frame buffer device [ 4.510879] input: Power Button as /devices/LNXSYSTM:00/device:00/PNP0C0C:00/input/input0 [ 4.519061] ACPI: Power Button [PWRB] [ 4.522784] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input1 [ 4.530186] ACPI: Power Button [PWRF] [ 4.535037] GHES: APEI firmware first mode is enabled by APEI bit and WHEA _OSC. [ 4.542520] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled [ 4.569719] 00:02: ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A [ 4.596256] 00:03: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A [ 4.602307] Non-volatile memory driver v1.3 [ 4.606532] Linux agpgart interface v0.103 [ 4.612249] crash memory driver: version 1.1 [ 4.616758] rdac: device handler registered [ 4.621003] hp_sw: device handler registered [ 4.625282] emc: device handler registered [ 4.629536] alua: device handler registered [ 4.633764] libphy: Fixed MDIO Bus: probed [ 4.637932] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver [ 4.644474] ehci-pci: EHCI PCI platform driver [ 4.648942] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver [ 4.655133] ohci-pci: OHCI PCI platform driver [ 4.659598] uhci_hcd: USB Universal Host Controller Interface driver [ 4.666061] xhci_hcd 0000:02:00.3: xHCI Host Controller [ 4.671350] xhci_hcd 0000:02:00.3: new USB bus registered, assigned bus number 1 [ 4.678855] xhci_hcd 0000:02:00.3: hcc params 0x0270f665 hci version 0x100 quirks 0x00000410 [ 4.687327] xhci_hcd 0000:02:00.3: irq 50 for MSI/MSI-X [ 4.687351] xhci_hcd 0000:02:00.3: irq 51 for MSI/MSI-X [ 4.687368] xhci_hcd 0000:02:00.3: irq 52 for MSI/MSI-X [ 4.687389] xhci_hcd 0000:02:00.3: irq 53 for MSI/MSI-X [ 4.687407] xhci_hcd 0000:02:00.3: irq 54 for MSI/MSI-X [ 4.687427] xhci_hcd 0000:02:00.3: irq 55 for MSI/MSI-X [ 4.687446] xhci_hcd 0000:02:00.3: irq 56 for MSI/MSI-X [ 4.687465] xhci_hcd 0000:02:00.3: irq 57 for MSI/MSI-X [ 4.687596] usb usb1: New USB device found, idVendor=1d6b, idProduct=0002 [ 4.694391] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1 [ 4.701620] usb usb1: Product: xHCI Host Controller [ 4.706508] usb usb1: Manufacturer: Linux 3.10.0-957.27.2.el7_lustre.pl2.x86_64 xhci-hcd [ 4.714603] usb usb1: SerialNumber: 0000:02:00.3 [ 4.719332] hub 1-0:1.0: USB hub found [ 4.723101] hub 1-0:1.0: 2 ports detected [ 4.727343] xhci_hcd 0000:02:00.3: xHCI Host Controller [ 4.732649] xhci_hcd 0000:02:00.3: new USB bus registered, assigned bus number 2 [ 4.740062] usb usb2: We don't know the algorithms for LPM for this host, disabling LPM. [ 4.748175] usb usb2: New USB device found, idVendor=1d6b, idProduct=0003 [ 4.754974] usb usb2: New USB device strings: Mfr=3, Product=2, SerialNumber=1 [ 4.762201] usb usb2: Product: xHCI Host Controller [ 4.767090] usb usb2: Manufacturer: Linux 3.10.0-957.27.2.el7_lustre.pl2.x86_64 xhci-hcd [ 4.775185] usb usb2: SerialNumber: 0000:02:00.3 [ 4.779909] hub 2-0:1.0: USB hub found [ 4.783674] hub 2-0:1.0: 2 ports detected [ 4.787986] xhci_hcd 0000:41:00.3: xHCI Host Controller [ 4.793295] xhci_hcd 0000:41:00.3: new USB bus registered, assigned bus number 3 [ 4.800807] xhci_hcd 0000:41:00.3: hcc params 0x0270f665 hci version 0x100 quirks 0x00000410 [ 4.809289] xhci_hcd 0000:41:00.3: irq 59 for MSI/MSI-X [ 4.809308] xhci_hcd 0000:41:00.3: irq 60 for MSI/MSI-X [ 4.809327] xhci_hcd 0000:41:00.3: irq 61 for MSI/MSI-X [ 4.809346] xhci_hcd 0000:41:00.3: irq 62 for MSI/MSI-X [ 4.809367] xhci_hcd 0000:41:00.3: irq 63 for MSI/MSI-X [ 4.809392] xhci_hcd 0000:41:00.3: irq 64 for MSI/MSI-X [ 4.809412] xhci_hcd 0000:41:00.3: irq 65 for MSI/MSI-X [ 4.809431] xhci_hcd 0000:41:00.3: irq 66 for MSI/MSI-X [ 4.809579] usb usb3: New USB device found, idVendor=1d6b, idProduct=0002 [ 4.816370] usb usb3: New USB device strings: Mfr=3, Product=2, SerialNumber=1 [ 4.823599] usb usb3: Product: xHCI Host Controller [ 4.828488] usb usb3: Manufacturer: Linux 3.10.0-957.27.2.el7_lustre.pl2.x86_64 xhci-hcd [ 4.836581] usb usb3: SerialNumber: 0000:41:00.3 [ 4.841315] hub 3-0:1.0: USB hub found [ 4.845079] hub 3-0:1.0: 2 ports detected [ 4.849333] xhci_hcd 0000:41:00.3: xHCI Host Controller [ 4.854614] xhci_hcd 0000:41:00.3: new USB bus registered, assigned bus number 4 [ 4.862060] usb usb4: We don't know the algorithms for LPM for this host, disabling LPM. [ 4.870172] usb usb4: New USB device found, idVendor=1d6b, idProduct=0003 [ 4.876970] usb usb4: New USB device strings: Mfr=3, Product=2, SerialNumber=1 [ 4.884199] usb usb4: Product: xHCI Host Controller [ 4.889087] usb usb4: Manufacturer: Linux 3.10.0-957.27.2.el7_lustre.pl2.x86_64 xhci-hcd [ 4.897182] usb usb4: SerialNumber: 0000:41:00.3 [ 4.901895] hub 4-0:1.0: USB hub found [ 4.905660] hub 4-0:1.0: 2 ports detected [ 4.909923] usbcore: registered new interface driver usbserial_generic [ 4.916461] usbserial: USB Serial support registered for generic [ 4.922513] i8042: PNP: No PS/2 controller found. Probing ports directly. [ 5.160615] usb 3-1: new high-speed USB device number 2 using xhci_hcd [ 5.290568] usb 3-1: New USB device found, idVendor=1604, idProduct=10c0 [ 5.297278] usb 3-1: New USB device strings: Mfr=0, Product=0, SerialNumber=0 [ 5.310212] hub 3-1:1.0: USB hub found [ 5.314196] hub 3-1:1.0: 4 ports detected [ 5.960416] i8042: No controller found [ 5.964184] sched: RT throttling activated [ 5.964193] tsc: Refined TSC clocksource calibration: 1996.249 MHz [ 5.964327] mousedev: PS/2 mouse device common for all mice [ 5.964539] rtc_cmos 00:01: RTC can wake from S4 [ 5.964925] rtc_cmos 00:01: rtc core: registered rtc_cmos as rtc0 [ 5.965025] rtc_cmos 00:01: alarms up to one month, y3k, 114 bytes nvram, hpet irqs [ 5.965079] cpuidle: using governor menu [ 5.965295] EFI Variables Facility v0.08 2004-May-17 [ 5.987788] hidraw: raw HID events driver (C) Jiri Kosina [ 5.987887] usbcore: registered new interface driver usbhid [ 5.987888] usbhid: USB HID core driver [ 5.987976] drop_monitor: Initializing network drop monitor service [ 5.988106] TCP: cubic registered [ 5.988110] Initializing XFRM netlink socket [ 5.988310] NET: Registered protocol family 10 [ 5.988773] NET: Registered protocol family 17 [ 5.988777] mpls_gso: MPLS GSO support [ 5.989847] mce: Using 23 MCE banks [ 5.989891] microcode: CPU0: patch_level=0x08001250 [ 5.989901] microcode: CPU1: patch_level=0x08001250 [ 5.989914] microcode: CPU2: patch_level=0x08001250 [ 5.989925] microcode: CPU3: patch_level=0x08001250 [ 5.989936] microcode: CPU4: patch_level=0x08001250 [ 5.989949] microcode: CPU5: patch_level=0x08001250 [ 5.989962] microcode: CPU6: patch_level=0x08001250 [ 5.989976] microcode: CPU7: patch_level=0x08001250 [ 5.989985] microcode: CPU8: patch_level=0x08001250 [ 5.989993] microcode: CPU9: patch_level=0x08001250 [ 5.990004] microcode: CPU10: patch_level=0x08001250 [ 5.990013] microcode: CPU11: patch_level=0x08001250 [ 5.990023] microcode: CPU12: patch_level=0x08001250 [ 5.990031] microcode: CPU13: patch_level=0x08001250 [ 5.990042] microcode: CPU14: patch_level=0x08001250 [ 5.990053] microcode: CPU15: patch_level=0x08001250 [ 5.993622] microcode: CPU16: patch_level=0x08001250 [ 5.993629] microcode: CPU17: patch_level=0x08001250 [ 5.993638] microcode: CPU18: patch_level=0x08001250 [ 5.993642] usb 3-1.1: new high-speed USB device number 3 using xhci_hcd [ 5.993647] microcode: CPU19: patch_level=0x08001250 [ 5.993655] microcode: CPU20: patch_level=0x08001250 [ 5.993666] microcode: CPU21: patch_level=0x08001250 [ 5.993676] microcode: CPU22: patch_level=0x08001250 [ 5.993687] microcode: CPU23: patch_level=0x08001250 [ 5.993695] microcode: CPU24: patch_level=0x08001250 [ 5.993709] microcode: CPU25: patch_level=0x08001250 [ 5.993719] microcode: CPU26: patch_level=0x08001250 [ 5.993729] microcode: CPU27: patch_level=0x08001250 [ 5.993737] microcode: CPU28: patch_level=0x08001250 [ 5.993745] microcode: CPU29: patch_level=0x08001250 [ 5.993754] microcode: CPU30: patch_level=0x08001250 [ 5.993764] microcode: CPU31: patch_level=0x08001250 [ 5.993772] microcode: CPU32: patch_level=0x08001250 [ 5.993781] microcode: CPU33: patch_level=0x08001250 [ 5.993792] microcode: CPU34: patch_level=0x08001250 [ 5.993799] microcode: CPU35: patch_level=0x08001250 [ 5.993808] microcode: CPU36: patch_level=0x08001250 [ 5.993816] microcode: CPU37: patch_level=0x08001250 [ 5.993825] microcode: CPU38: patch_level=0x08001250 [ 5.993832] microcode: CPU39: patch_level=0x08001250 [ 5.993840] microcode: CPU40: patch_level=0x08001250 [ 5.993848] microcode: CPU41: patch_level=0x08001250 [ 5.993856] microcode: CPU42: patch_level=0x08001250 [ 5.993864] microcode: CPU43: patch_level=0x08001250 [ 5.993872] microcode: CPU44: patch_level=0x08001250 [ 5.993880] microcode: CPU45: patch_level=0x08001250 [ 5.993891] microcode: CPU46: patch_level=0x08001250 [ 5.993899] microcode: CPU47: patch_level=0x08001250 [ 5.993953] microcode: Microcode Update Driver: v2.01 , Peter Oruba [ 5.994107] PM: Hibernation image not present or could not be loaded. [ 5.994109] Loading compiled-in X.509 certificates [ 5.994133] Loaded X.509 cert 'CentOS Linux kpatch signing key: ea0413152cde1d98ebdca3fe6f0230904c9ef717' [ 5.994147] Loaded X.509 cert 'CentOS Linux Driver update signing key: 7f421ee0ab69461574bb358861dbe77762a4201b' [ 5.994526] Loaded X.509 cert 'CentOS Linux kernel signing key: 468656045a39b52ff2152c315f6198c3e658f24d' [ 5.994539] registered taskstats version 1 [ 5.996601] Key type trusted registered [ 5.998127] Key type encrypted registered [ 5.998172] IMA: No TPM chip found, activating TPM-bypass! (rc=-19) [ 6.000138] Magic number: 12:127:388 [ 6.000141] machinecheck machinecheck36: hash matches [ 6.000154] thermal cooling_device3: hash matches [ 6.000169] clockevents clockevent105: hash matches [ 6.000224] platform ACPI0007:38: hash matches [ 6.000239] acpi ACPI0007:38: hash matches [ 6.000243] acpi ACPI0007:0b: hash matches [ 6.000282] memory memory1905: hash matches [ 6.000302] memory memory1459: hash matches [ 6.007685] rtc_cmos 00:01: setting system clock to 2020-07-08 16:23:56 UTC (1594225436) [ 6.414918] Switched to clocksource tsc [ 6.419844] Freeing unused kernel memory: 1876k freed [ 6.425077] Write protecting the kernel read-only data: 12288k [ 6.426598] usb 3-1.1: New USB device found, idVendor=1604, idProduct=10c0 [ 6.426599] usb 3-1.1: New USB device strings: Mfr=0, Product=0, SerialNumber=0 [ 6.430237] hub 3-1.1:1.0: USB hub found [ 6.430596] hub 3-1.1:1.0: 4 ports detected [ 6.454616] Freeing unused kernel memory: 504k freed [ 6.460967] Freeing unused kernel memory: 596k freed [ 6.494944] usb 3-1.4: new high-speed USB device number 4 using xhci_hcd [ 6.516572] systemd[1]: systemd 219 running in system mode. (+PAM +AUDIT +SELINUX +IMA -APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 -SECCOMP +BLKID +ELFUTILS +KMOD +IDN) [ 6.530639] usb 1-1: new high-speed USB device number 2 using xhci_hcd [ 6.542205] systemd[1]: Detected architecture x86-64. [ 6.547265] systemd[1]: Running in initial RAM disk. [ 6.560770] systemd[1]: Set hostname to . [ 6.581727] usb 3-1.4: New USB device found, idVendor=1604, idProduct=10c0 [ 6.588613] usb 3-1.4: New USB device strings: Mfr=0, Product=0, SerialNumber=0 [ 6.601530] systemd[1]: Reached target Timers. [ 6.610723] systemd[1]: Reached target Local File Systems. [ 6.621907] systemd[1]: Created slice Root Slice. [ 6.622248] hub 3-1.4:1.0: USB hub found [ 6.622601] hub 3-1.4:1.0: 4 ports detected [ 6.640755] systemd[1]: Created slice System Slice. [ 6.651749] systemd[1]: Listening on Journal Socket. [ 6.659489] usb 1-1: New USB device found, idVendor=0424, idProduct=2744 [ 6.666313] usb 1-1: New USB device strings: Mfr=1, Product=2, SerialNumber=0 [ 6.674831] usb 1-1: Product: USB2734 [ 6.678497] usb 1-1: Manufacturer: Microchip Tech [ 6.683737] systemd[1]: Starting Journal Service... [ 6.692269] hub 1-1:1.0: USB hub found [ 6.696240] hub 1-1:1.0: 4 ports detected [ 6.696595] systemd[1]: Starting Setup Virtual Console... [ 6.722661] systemd[1]: Starting Create list of required static device nodes for the current kernel... [ 6.742166] systemd[1]: Starting dracut cmdline hook... [ 6.751710] systemd[1]: Reached target Slices. [ 6.760734] systemd[1]: Listening on udev Control Socket. [ 6.771724] systemd[1]: Listening on udev Kernel Socket. [ 6.782700] systemd[1]: Reached target Sockets. [ 6.787251] usb 2-1: new SuperSpeed USB device number 2 using xhci_hcd [ 6.799103] systemd[1]: Starting Apply Kernel Variables... [ 6.805992] usb 2-1: New USB device found, idVendor=0424, idProduct=5744 [ 6.812828] usb 2-1: New USB device strings: Mfr=2, Product=3, SerialNumber=0 [ 6.821344] usb 2-1: Product: USB5734 [ 6.826139] usb 2-1: Manufacturer: Microchip Tech [ 6.830902] systemd[1]: Reached target Swap. [ 6.836271] hub 2-1:1.0: USB hub found [ 6.840619] hub 2-1:1.0: 4 ports detected [ 6.846807] usb: port power management may be unreliable [ 6.852452] systemd[1]: Started Journal Service. [ 6.997221] pps_core: LinuxPPS API ver. 1 registered [ 7.002199] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti [ 7.015066] PTP clock support registered [ 7.021133] mlx_compat: loading out-of-tree module taints kernel. [ 7.021137] megasas: 07.705.02.00-rh1 [ 7.021432] megaraid_sas 0000:c1:00.0: FW now in Ready state [ 7.021437] megaraid_sas 0000:c1:00.0: 64 bit DMA mask and 32 bit consistent mask [ 7.021743] megaraid_sas 0000:c1:00.0: irq 68 for MSI/MSI-X [ 7.021767] megaraid_sas 0000:c1:00.0: irq 69 for MSI/MSI-X [ 7.021793] megaraid_sas 0000:c1:00.0: irq 70 for MSI/MSI-X [ 7.021815] megaraid_sas 0000:c1:00.0: irq 71 for MSI/MSI-X [ 7.021838] megaraid_sas 0000:c1:00.0: irq 72 for MSI/MSI-X [ 7.021861] megaraid_sas 0000:c1:00.0: irq 73 for MSI/MSI-X [ 7.021883] megaraid_sas 0000:c1:00.0: irq 74 for MSI/MSI-X [ 7.021906] megaraid_sas 0000:c1:00.0: irq 75 for MSI/MSI-X [ 7.021928] megaraid_sas 0000:c1:00.0: irq 76 for MSI/MSI-X [ 7.021950] megaraid_sas 0000:c1:00.0: irq 77 for MSI/MSI-X [ 7.021971] megaraid_sas 0000:c1:00.0: irq 78 for MSI/MSI-X [ 7.021992] megaraid_sas 0000:c1:00.0: irq 79 for MSI/MSI-X [ 7.022018] megaraid_sas 0000:c1:00.0: irq 80 for MSI/MSI-X [ 7.022040] megaraid_sas 0000:c1:00.0: irq 81 for MSI/MSI-X [ 7.022062] megaraid_sas 0000:c1:00.0: irq 82 for MSI/MSI-X [ 7.022084] megaraid_sas 0000:c1:00.0: irq 83 for MSI/MSI-X [ 7.022105] megaraid_sas 0000:c1:00.0: irq 84 for MSI/MSI-X [ 7.022127] megaraid_sas 0000:c1:00.0: irq 85 for MSI/MSI-X [ 7.022149] megaraid_sas 0000:c1:00.0: irq 86 for MSI/MSI-X [ 7.022170] megaraid_sas 0000:c1:00.0: irq 87 for MSI/MSI-X [ 7.022191] megaraid_sas 0000:c1:00.0: irq 88 for MSI/MSI-X [ 7.022213] megaraid_sas 0000:c1:00.0: irq 89 for MSI/MSI-X [ 7.022234] megaraid_sas 0000:c1:00.0: irq 90 for MSI/MSI-X [ 7.022255] megaraid_sas 0000:c1:00.0: irq 91 for MSI/MSI-X [ 7.022283] megaraid_sas 0000:c1:00.0: irq 92 for MSI/MSI-X [ 7.022306] megaraid_sas 0000:c1:00.0: irq 93 for MSI/MSI-X [ 7.022328] megaraid_sas 0000:c1:00.0: irq 94 for MSI/MSI-X [ 7.022351] megaraid_sas 0000:c1:00.0: irq 95 for MSI/MSI-X [ 7.022373] megaraid_sas 0000:c1:00.0: irq 96 for MSI/MSI-X [ 7.022395] megaraid_sas 0000:c1:00.0: irq 97 for MSI/MSI-X [ 7.022417] megaraid_sas 0000:c1:00.0: irq 98 for MSI/MSI-X [ 7.022438] megaraid_sas 0000:c1:00.0: irq 99 for MSI/MSI-X [ 7.022460] megaraid_sas 0000:c1:00.0: irq 100 for MSI/MSI-X [ 7.022483] megaraid_sas 0000:c1:00.0: irq 101 for MSI/MSI-X [ 7.022505] megaraid_sas 0000:c1:00.0: irq 102 for MSI/MSI-X [ 7.022527] megaraid_sas 0000:c1:00.0: irq 103 for MSI/MSI-X [ 7.022549] megaraid_sas 0000:c1:00.0: irq 104 for MSI/MSI-X [ 7.022570] megaraid_sas 0000:c1:00.0: irq 105 for MSI/MSI-X [ 7.022591] megaraid_sas 0000:c1:00.0: irq 106 for MSI/MSI-X [ 7.022612] megaraid_sas 0000:c1:00.0: irq 107 for MSI/MSI-X [ 7.022633] megaraid_sas 0000:c1:00.0: irq 108 for MSI/MSI-X [ 7.022659] megaraid_sas 0000:c1:00.0: irq 109 for MSI/MSI-X [ 7.022681] megaraid_sas 0000:c1:00.0: irq 110 for MSI/MSI-X [ 7.022701] megaraid_sas 0000:c1:00.0: irq 111 for MSI/MSI-X [ 7.022722] megaraid_sas 0000:c1:00.0: irq 112 for MSI/MSI-X [ 7.022743] megaraid_sas 0000:c1:00.0: irq 113 for MSI/MSI-X [ 7.022765] megaraid_sas 0000:c1:00.0: irq 114 for MSI/MSI-X [ 7.022786] megaraid_sas 0000:c1:00.0: irq 115 for MSI/MSI-X [ 7.022911] megaraid_sas 0000:c1:00.0: firmware supports msix : (96) [ 7.022913] megaraid_sas 0000:c1:00.0: current msix/online cpus : (48/48) [ 7.022914] megaraid_sas 0000:c1:00.0: RDPQ mode : (disabled) [ 7.022917] megaraid_sas 0000:c1:00.0: Current firmware supports maximum commands: 928 LDIO threshold: 237 [ 7.023174] megaraid_sas 0000:c1:00.0: Configured max firmware commands: 927 [ 7.025157] megaraid_sas 0000:c1:00.0: FW supports sync cache : No [ 7.039980] mpt3sas: loading out-of-tree module taints kernel. [ 7.075563] mpt3sas: module verification failed: signature and/or required key missing - tainting kernel [ 7.105391] Compat-mlnx-ofed backport release: 1c4bf42 [ 7.105657] tg3.c:v3.137 (May 11, 2014) [ 7.105729] libata version 3.00 loaded. [ 7.115941] Backport based on mlnx_ofed/mlnx-ofa_kernel-4.0.git 1c4bf42 [ 7.117412] tg3 0000:81:00.0 eth0: Tigon3 [partno(BCM95720) rev 5720000] (PCI Express) MAC address 4c:d9:8f:48:5a:c7 [ 7.117415] tg3 0000:81:00.0 eth0: attached PHY is 5720C (10/100/1000Base-T Ethernet) (WireSpeed[1], EEE[1]) [ 7.117417] tg3 0000:81:00.0 eth0: RXcsums[1] LinkChgREG[0] MIirq[0] ASF[1] TSOcap[1] [ 7.117419] tg3 0000:81:00.0 eth0: dma_rwctrl[00000001] dma_mask[64-bit] [ 7.150177] tg3 0000:81:00.1 eth1: Tigon3 [partno(BCM95720) rev 5720000] (PCI Express) MAC address 4c:d9:8f:48:5a:c8 [ 7.150181] tg3 0000:81:00.1 eth1: attached PHY is 5720C (10/100/1000Base-T Ethernet) (WireSpeed[1], EEE[1]) [ 7.150183] tg3 0000:81:00.1 eth1: RXcsums[1] LinkChgREG[0] MIirq[0] ASF[1] TSOcap[1] [ 7.150185] tg3 0000:81:00.1 eth1: dma_rwctrl[00000001] dma_mask[64-bit] [ 7.186680] compat.git: mlnx_ofed/mlnx-ofa_kernel-4.0.git [ 7.209463] mpt3sas version 31.00.00.00 loaded [ 7.217036] mpt3sas_cm0: 63 BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (263564440 kB) [ 7.225915] ahci 0000:86:00.2: version 3.0 [ 7.226345] ahci 0000:86:00.2: irq 120 for MSI/MSI-X [ 7.226352] ahci 0000:86:00.2: irq 121 for MSI/MSI-X [ 7.226357] ahci 0000:86:00.2: irq 122 for MSI/MSI-X [ 7.226362] ahci 0000:86:00.2: irq 123 for MSI/MSI-X [ 7.226367] ahci 0000:86:00.2: irq 124 for MSI/MSI-X [ 7.226371] ahci 0000:86:00.2: irq 125 for MSI/MSI-X [ 7.226376] ahci 0000:86:00.2: irq 126 for MSI/MSI-X [ 7.226380] ahci 0000:86:00.2: irq 127 for MSI/MSI-X [ 7.226384] ahci 0000:86:00.2: irq 128 for MSI/MSI-X [ 7.226389] ahci 0000:86:00.2: irq 129 for MSI/MSI-X [ 7.226393] ahci 0000:86:00.2: irq 130 for MSI/MSI-X [ 7.226398] ahci 0000:86:00.2: irq 131 for MSI/MSI-X [ 7.226402] ahci 0000:86:00.2: irq 132 for MSI/MSI-X [ 7.226407] ahci 0000:86:00.2: irq 133 for MSI/MSI-X [ 7.226411] ahci 0000:86:00.2: irq 134 for MSI/MSI-X [ 7.226416] ahci 0000:86:00.2: irq 135 for MSI/MSI-X [ 7.226992] ahci 0000:86:00.2: AHCI 0001.0301 32 slots 1 ports 6 Gbps 0x1 impl SATA mode [ 7.235090] ahci 0000:86:00.2: flags: 64bit ncq sntf ilck pm led clo only pmp fbs pio slum part [ 7.244795] scsi host2: ahci [ 7.249332] ata1: SATA max UDMA/133 abar m4096@0xc0a02000 port 0xc0a02100 irq 120 [ 7.250149] mlx5_core 0000:01:00.0: firmware version: 20.26.1040 [ 7.250178] mlx5_core 0000:01:00.0: 126.016 Gb/s available PCIe bandwidth, limited by 8 GT/s x16 link at 0000:00:03.1 (capable of 252.048 Gb/s with 16 GT/s x16 link) [ 7.347909] mpt3sas_cm0: IOC Number : 0 [ 7.348400] mpt3sas 0000:84:00.0: irq 137 for MSI/MSI-X [ 7.348462] mpt3sas 0000:84:00.0: irq 138 for MSI/MSI-X [ 7.348523] mpt3sas 0000:84:00.0: irq 139 for MSI/MSI-X [ 7.348583] mpt3sas 0000:84:00.0: irq 140 for MSI/MSI-X [ 7.348643] mpt3sas 0000:84:00.0: irq 141 for MSI/MSI-X [ 7.348677] mpt3sas 0000:84:00.0: irq 142 for MSI/MSI-X [ 7.348696] mpt3sas 0000:84:00.0: irq 143 for MSI/MSI-X [ 7.348762] mpt3sas 0000:84:00.0: irq 144 for MSI/MSI-X [ 7.348823] mpt3sas 0000:84:00.0: irq 145 for MSI/MSI-X [ 7.348884] mpt3sas 0000:84:00.0: irq 146 for MSI/MSI-X [ 7.348945] mpt3sas 0000:84:00.0: irq 147 for MSI/MSI-X [ 7.349005] mpt3sas 0000:84:00.0: irq 148 for MSI/MSI-X [ 7.349066] mpt3sas 0000:84:00.0: irq 149 for MSI/MSI-X [ 7.349126] mpt3sas 0000:84:00.0: irq 150 for MSI/MSI-X [ 7.349187] mpt3sas 0000:84:00.0: irq 151 for MSI/MSI-X [ 7.349250] mpt3sas 0000:84:00.0: irq 152 for MSI/MSI-X [ 7.349310] mpt3sas 0000:84:00.0: irq 153 for MSI/MSI-X [ 7.349370] mpt3sas 0000:84:00.0: irq 154 for MSI/MSI-X [ 7.349430] mpt3sas 0000:84:00.0: irq 155 for MSI/MSI-X [ 7.349490] mpt3sas 0000:84:00.0: irq 156 for MSI/MSI-X [ 7.349551] mpt3sas 0000:84:00.0: irq 157 for MSI/MSI-X [ 7.349610] mpt3sas 0000:84:00.0: irq 158 for MSI/MSI-X [ 7.349671] mpt3sas 0000:84:00.0: irq 159 for MSI/MSI-X [ 7.349697] mpt3sas 0000:84:00.0: irq 160 for MSI/MSI-X [ 7.349756] mpt3sas 0000:84:00.0: irq 161 for MSI/MSI-X [ 7.349816] mpt3sas 0000:84:00.0: irq 162 for MSI/MSI-X [ 7.349875] mpt3sas 0000:84:00.0: irq 163 for MSI/MSI-X [ 7.349934] mpt3sas 0000:84:00.0: irq 164 for MSI/MSI-X [ 7.349993] mpt3sas 0000:84:00.0: irq 165 for MSI/MSI-X [ 7.350052] mpt3sas 0000:84:00.0: irq 166 for MSI/MSI-X [ 7.350111] mpt3sas 0000:84:00.0: irq 167 for MSI/MSI-X [ 7.350171] mpt3sas 0000:84:00.0: irq 168 for MSI/MSI-X [ 7.350230] mpt3sas 0000:84:00.0: irq 169 for MSI/MSI-X [ 7.350290] mpt3sas 0000:84:00.0: irq 170 for MSI/MSI-X [ 7.350349] mpt3sas 0000:84:00.0: irq 171 for MSI/MSI-X [ 7.350409] mpt3sas 0000:84:00.0: irq 172 for MSI/MSI-X [ 7.350469] mpt3sas 0000:84:00.0: irq 173 for MSI/MSI-X [ 7.350528] mpt3sas 0000:84:00.0: irq 174 for MSI/MSI-X [ 7.350588] mpt3sas 0000:84:00.0: irq 175 for MSI/MSI-X [ 7.350649] mpt3sas 0000:84:00.0: irq 176 for MSI/MSI-X [ 7.350676] mpt3sas 0000:84:00.0: irq 177 for MSI/MSI-X [ 7.350735] mpt3sas 0000:84:00.0: irq 178 for MSI/MSI-X [ 7.350794] mpt3sas 0000:84:00.0: irq 179 for MSI/MSI-X [ 7.350855] mpt3sas 0000:84:00.0: irq 180 for MSI/MSI-X [ 7.350915] mpt3sas 0000:84:00.0: irq 181 for MSI/MSI-X [ 7.350974] mpt3sas 0000:84:00.0: irq 182 for MSI/MSI-X [ 7.351034] mpt3sas 0000:84:00.0: irq 183 for MSI/MSI-X [ 7.351095] mpt3sas 0000:84:00.0: irq 184 for MSI/MSI-X [ 7.354822] mpt3sas0-msix0: PCI-MSI-X enabled: IRQ 137 [ 7.354822] mpt3sas0-msix1: PCI-MSI-X enabled: IRQ 138 [ 7.354823] mpt3sas0-msix2: PCI-MSI-X enabled: IRQ 139 [ 7.354823] mpt3sas0-msix3: PCI-MSI-X enabled: IRQ 140 [ 7.354824] mpt3sas0-msix4: PCI-MSI-X enabled: IRQ 141 [ 7.354824] mpt3sas0-msix5: PCI-MSI-X enabled: IRQ 142 [ 7.354825] mpt3sas0-msix6: PCI-MSI-X enabled: IRQ 143 [ 7.354825] mpt3sas0-msix7: PCI-MSI-X enabled: IRQ 144 [ 7.354826] mpt3sas0-msix8: PCI-MSI-X enabled: IRQ 145 [ 7.354826] mpt3sas0-msix9: PCI-MSI-X enabled: IRQ 146 [ 7.354827] mpt3sas0-msix10: PCI-MSI-X enabled: IRQ 147 [ 7.354827] mpt3sas0-msix11: PCI-MSI-X enabled: IRQ 148 [ 7.354828] mpt3sas0-msix12: PCI-MSI-X enabled: IRQ 149 [ 7.354828] mpt3sas0-msix13: PCI-MSI-X enabled: IRQ 150 [ 7.354829] mpt3sas0-msix14: PCI-MSI-X enabled: IRQ 151 [ 7.354829] mpt3sas0-msix15: PCI-MSI-X enabled: IRQ 152 [ 7.354830] mpt3sas0-msix16: PCI-MSI-X enabled: IRQ 153 [ 7.354830] mpt3sas0-msix17: PCI-MSI-X enabled: IRQ 154 [ 7.354830] mpt3sas0-msix18: PCI-MSI-X enabled: IRQ 155 [ 7.354831] mpt3sas0-msix19: PCI-MSI-X enabled: IRQ 156 [ 7.354831] mpt3sas0-msix20: PCI-MSI-X enabled: IRQ 157 [ 7.354832] mpt3sas0-msix21: PCI-MSI-X enabled: IRQ 158 [ 7.354832] mpt3sas0-msix22: PCI-MSI-X enabled: IRQ 159 [ 7.354833] mpt3sas0-msix23: PCI-MSI-X enabled: IRQ 160 [ 7.354833] mpt3sas0-msix24: PCI-MSI-X enabled: IRQ 161 [ 7.354834] mpt3sas0-msix25: PCI-MSI-X enabled: IRQ 162 [ 7.354834] mpt3sas0-msix26: PCI-MSI-X enabled: IRQ 163 [ 7.354835] mpt3sas0-msix27: PCI-MSI-X enabled: IRQ 164 [ 7.354835] mpt3sas0-msix28: PCI-MSI-X enabled: IRQ 165 [ 7.354836] mpt3sas0-msix29: PCI-MSI-X enabled: IRQ 166 [ 7.354836] mpt3sas0-msix30: PCI-MSI-X enabled: IRQ 167 [ 7.354837] mpt3sas0-msix31: PCI-MSI-X enabled: IRQ 168 [ 7.354837] mpt3sas0-msix32: PCI-MSI-X enabled: IRQ 169 [ 7.354837] mpt3sas0-msix33: PCI-MSI-X enabled: IRQ 170 [ 7.354838] mpt3sas0-msix34: PCI-MSI-X enabled: IRQ 171 [ 7.354838] mpt3sas0-msix35: PCI-MSI-X enabled: IRQ 172 [ 7.354839] mpt3sas0-msix36: PCI-MSI-X enabled: IRQ 173 [ 7.354839] mpt3sas0-msix37: PCI-MSI-X enabled: IRQ 174 [ 7.354840] mpt3sas0-msix38: PCI-MSI-X enabled: IRQ 175 [ 7.354840] mpt3sas0-msix39: PCI-MSI-X enabled: IRQ 176 [ 7.354841] mpt3sas0-msix40: PCI-MSI-X enabled: IRQ 177 [ 7.354841] mpt3sas0-msix41: PCI-MSI-X enabled: IRQ 178 [ 7.354842] mpt3sas0-msix42: PCI-MSI-X enabled: IRQ 179 [ 7.354842] mpt3sas0-msix43: PCI-MSI-X enabled: IRQ 180 [ 7.354843] mpt3sas0-msix44: PCI-MSI-X enabled: IRQ 181 [ 7.354843] mpt3sas0-msix45: PCI-MSI-X enabled: IRQ 182 [ 7.354843] mpt3sas0-msix46: PCI-MSI-X enabled: IRQ 183 [ 7.354844] mpt3sas0-msix47: PCI-MSI-X enabled: IRQ 184 [ 7.354845] mpt3sas_cm0: iomem(0x00000000ac000000), mapped(0xffffa9309a200000), size(1048576) [ 7.354846] mpt3sas_cm0: ioport(0x0000000000008000), size(256) [ 7.390661] megaraid_sas 0000:c1:00.0: Init cmd return status SUCCESS for SCSI host 0 [ 7.416660] megaraid_sas 0000:c1:00.0: firmware type : Legacy(64 VD) firmware [ 7.416662] megaraid_sas 0000:c1:00.0: controller type : iMR(0MB) [ 7.416663] megaraid_sas 0000:c1:00.0: Online Controller Reset(OCR) : Enabled [ 7.416664] megaraid_sas 0000:c1:00.0: Secure JBOD support : No [ 7.416665] megaraid_sas 0000:c1:00.0: NVMe passthru support : No [ 7.438190] megaraid_sas 0000:c1:00.0: INIT adapter done [ 7.438193] megaraid_sas 0000:c1:00.0: Jbod map is not supported megasas_setup_jbod_map 5146 [ 7.464497] megaraid_sas 0000:c1:00.0: pci id : (0x1000)/(0x005f)/(0x1028)/(0x1f4b) [ 7.464499] megaraid_sas 0000:c1:00.0: unevenspan support : yes [ 7.464500] megaraid_sas 0000:c1:00.0: firmware crash dump : no [ 7.464501] megaraid_sas 0000:c1:00.0: jbod sync map : no [ 7.464506] scsi host0: Avago SAS based MegaRAID driver [ 7.484588] scsi 0:2:0:0: Direct-Access DELL PERC H330 Mini 4.30 PQ: 0 ANSI: 5 [ 7.484663] mpt3sas_cm0: IOC Number : 0 [ 7.484666] mpt3sas_cm0: sending message unit reset !! [ 7.489661] mpt3sas_cm0: message unit reset: SUCCESS [ 7.511301] mlx5_core 0000:01:00.0: irq 185 for MSI/MSI-X [ 7.511321] mlx5_core 0000:01:00.0: irq 186 for MSI/MSI-X [ 7.511340] mlx5_core 0000:01:00.0: irq 187 for MSI/MSI-X [ 7.511359] mlx5_core 0000:01:00.0: irq 188 for MSI/MSI-X [ 7.511378] mlx5_core 0000:01:00.0: irq 189 for MSI/MSI-X [ 7.511399] mlx5_core 0000:01:00.0: irq 190 for MSI/MSI-X [ 7.511418] mlx5_core 0000:01:00.0: irq 191 for MSI/MSI-X [ 7.511438] mlx5_core 0000:01:00.0: irq 192 for MSI/MSI-X [ 7.511460] mlx5_core 0000:01:00.0: irq 193 for MSI/MSI-X [ 7.511479] mlx5_core 0000:01:00.0: irq 194 for MSI/MSI-X [ 7.511497] mlx5_core 0000:01:00.0: irq 195 for MSI/MSI-X [ 7.511514] mlx5_core 0000:01:00.0: irq 196 for MSI/MSI-X [ 7.511533] mlx5_core 0000:01:00.0: irq 197 for MSI/MSI-X [ 7.511552] mlx5_core 0000:01:00.0: irq 198 for MSI/MSI-X [ 7.511570] mlx5_core 0000:01:00.0: irq 199 for MSI/MSI-X [ 7.511588] mlx5_core 0000:01:00.0: irq 200 for MSI/MSI-X [ 7.511607] mlx5_core 0000:01:00.0: irq 201 for MSI/MSI-X [ 7.511624] mlx5_core 0000:01:00.0: irq 202 for MSI/MSI-X [ 7.511644] mlx5_core 0000:01:00.0: irq 203 for MSI/MSI-X [ 7.511703] mlx5_core 0000:01:00.0: irq 204 for MSI/MSI-X [ 7.511722] mlx5_core 0000:01:00.0: irq 205 for MSI/MSI-X [ 7.511741] mlx5_core 0000:01:00.0: irq 206 for MSI/MSI-X [ 7.511760] mlx5_core 0000:01:00.0: irq 207 for MSI/MSI-X [ 7.511778] mlx5_core 0000:01:00.0: irq 208 for MSI/MSI-X [ 7.511797] mlx5_core 0000:01:00.0: irq 209 for MSI/MSI-X [ 7.511815] mlx5_core 0000:01:00.0: irq 210 for MSI/MSI-X [ 7.511833] mlx5_core 0000:01:00.0: irq 211 for MSI/MSI-X [ 7.511850] mlx5_core 0000:01:00.0: irq 212 for MSI/MSI-X [ 7.511871] mlx5_core 0000:01:00.0: irq 213 for MSI/MSI-X [ 7.511890] mlx5_core 0000:01:00.0: irq 214 for MSI/MSI-X [ 7.511908] mlx5_core 0000:01:00.0: irq 215 for MSI/MSI-X [ 7.511927] mlx5_core 0000:01:00.0: irq 216 for MSI/MSI-X [ 7.511945] mlx5_core 0000:01:00.0: irq 217 for MSI/MSI-X [ 7.511964] mlx5_core 0000:01:00.0: irq 218 for MSI/MSI-X [ 7.511986] mlx5_core 0000:01:00.0: irq 219 for MSI/MSI-X [ 7.512004] mlx5_core 0000:01:00.0: irq 220 for MSI/MSI-X [ 7.512021] mlx5_core 0000:01:00.0: irq 221 for MSI/MSI-X [ 7.512040] mlx5_core 0000:01:00.0: irq 222 for MSI/MSI-X [ 7.512060] mlx5_core 0000:01:00.0: irq 223 for MSI/MSI-X [ 7.512078] mlx5_core 0000:01:00.0: irq 224 for MSI/MSI-X [ 7.512096] mlx5_core 0000:01:00.0: irq 225 for MSI/MSI-X [ 7.512114] mlx5_core 0000:01:00.0: irq 226 for MSI/MSI-X [ 7.512132] mlx5_core 0000:01:00.0: irq 227 for MSI/MSI-X [ 7.512151] mlx5_core 0000:01:00.0: irq 228 for MSI/MSI-X [ 7.512169] mlx5_core 0000:01:00.0: irq 229 for MSI/MSI-X [ 7.512188] mlx5_core 0000:01:00.0: irq 230 for MSI/MSI-X [ 7.512207] mlx5_core 0000:01:00.0: irq 231 for MSI/MSI-X [ 7.512226] mlx5_core 0000:01:00.0: irq 232 for MSI/MSI-X [ 7.512245] mlx5_core 0000:01:00.0: irq 233 for MSI/MSI-X [ 7.513250] mlx5_core 0000:01:00.0: Port module event: module 0, Cable plugged [ 7.513530] mlx5_core 0000:01:00.0: mlx5_pcie_event:303:(pid 316): PCIe slot advertised sufficient power (27W). [ 7.520667] mlx5_core 0000:01:00.0: mlx5_fw_tracer_start:776:(pid 300): FWTracer: Ownership granted and active [ 7.589682] ata1: SATA link down (SStatus 0 SControl 300) [ 7.718805] mpt3sas_cm0: Allocated physical memory: size(38831 kB) [ 7.718806] mpt3sas_cm0: Current Controller Queue Depth(7564), Max Controller Queue Depth(7680) [ 7.718807] mpt3sas_cm0: Scatter Gather Elements per IO(128) [ 7.820123] mlx5_ib: Mellanox Connect-IB Infiniband driver v4.7-1.0.0 [ 7.853759] sd 0:2:0:0: [sda] 467664896 512-byte logical blocks: (239 GB/223 GiB) [ 7.861446] sd 0:2:0:0: [sda] Write Protect is off [ 7.866247] sd 0:2:0:0: [sda] Mode Sense: 1f 00 10 08 [ 7.866295] sd 0:2:0:0: [sda] Write cache: disabled, read cache: disabled, supports DPO and FUA [ 7.877186] sda: sda1 sda2 sda3 [ 7.880883] sd 0:2:0:0: [sda] Attached SCSI disk [ 7.920281] mpt3sas_cm0: FW Package Version(12.00.00.00) [ 7.926153] mpt3sas_cm0: SAS3616: FWVersion(12.00.00.00), ChipRevision(0x02), BiosVersion(00.00.00.00) [ 7.935481] mpt3sas_cm0: Protocol=(Initiator,Target,NVMe), Capabilities=(TLR,EEDP,Diag Trace Buffer,Task Set Full,NCQ) [ 7.954353] mpt3sas 0000:84:00.0: Enabled Extended Tags as Controller Supports [ 7.961596] mpt3sas_cm0: : host protection capabilities enabled DIF1 DIF2 DIF3 [ 7.968916] scsi host1: Fusion MPT SAS Host [ 7.973347] mpt3sas_cm0: registering trace buffer support [ 7.983048] mpt3sas_cm0: Trace buffer memory 2048 KB allocated [ 7.983466] random: crng init done [ 7.994802] mpt3sas_cm0: sending port enable !! [ 8.000998] mpt3sas_cm0: hba_port entry: ffff8eebed3d1f80, port: 255 is added to hba_port list [ 8.013629] mpt3sas_cm0: host_add: handle(0x0001), sas_addr(0x500605b00deb4820), phys(21) [ 8.023748] mpt3sas_cm0: detecting: handle(0x0018), sas_address(0x500a0984dfa20c24), phy(0) [ 8.032764] mpt3sas_cm0: REPORT_LUNS: handle(0x0018), retries(0) [ 8.041052] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0018), lun(0) [ 8.048123] scsi 1:0:0:0: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.057116] scsi 1:0:0:0: SSP: handle(0x0018), sas_addr(0x500a0984dfa20c24), phy(0), device_name(0x500a0984dfa20c24) [ 8.068311] scsi 1:0:0:0: enclosure logical id(0x300605b00d114820), slot(13) [ 8.075444] scsi 1:0:0:0: enclosure level(0x0000), connector name( C3 ) [ 8.082163] scsi 1:0:0:0: serial_number(021825001558 ) [ 8.087565] scsi 1:0:0:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.102325] EXT4-fs (sda2): mounted filesystem with ordered data mode. Opts: (null) [ 8.120650] scsi 1:0:0:1: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.129726] scsi 1:0:0:1: SSP: handle(0x0018), sas_addr(0x500a0984dfa20c24), phy(0), device_name(0x500a0984dfa20c24) [ 8.141540] scsi 1:0:0:1: enclosure logical id(0x300605b00d114820), slot(13) [ 8.150061] scsi 1:0:0:1: enclosure level(0x0000), connector name( C3 ) [ 8.150087] scsi 1:0:0:1: serial_number(021825001558 ) [ 8.150089] scsi 1:0:0:1: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.183900] scsi 1:0:0:31: Direct-Access DELL Universal Xport 0825 PQ: 0 ANSI: 5 [ 8.193361] scsi 1:0:0:31: SSP: handle(0x0018), sas_addr(0x500a0984dfa20c24), phy(0), device_name(0x500a0984dfa20c24) [ 8.205267] scsi 1:0:0:31: enclosure logical id(0x300605b00d114820), slot(13) [ 8.212486] scsi 1:0:0:31: enclosure level(0x0000), connector name( C3 ) [ 8.220685] scsi 1:0:0:31: serial_number(021825001558 ) [ 8.227534] scsi 1:0:0:31: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.250946] mpt3sas_cm0: detecting: handle(0x0019), sas_address(0x500a0984dfa1fa10), phy(4) [ 8.259304] mpt3sas_cm0: REPORT_LUNS: handle(0x0019), retries(0) [ 8.266213] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0019), lun(0) [ 8.272755] scsi 1:0:1:0: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.280936] scsi 1:0:1:0: SSP: handle(0x0019), sas_addr(0x500a0984dfa1fa10), phy(4), device_name(0x500a0984dfa1fa10) [ 8.291449] scsi 1:0:1:0: enclosure logical id(0x300605b00d114820), slot(9) [ 8.298497] scsi 1:0:1:0: enclosure level(0x0000), connector name( C2 ) [ 8.305217] scsi 1:0:1:0: serial_number(021825001369 ) [ 8.310619] scsi 1:0:1:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.332688] scsi 1:0:1:1: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.341664] scsi 1:0:1:1: SSP: handle(0x0019), sas_addr(0x500a0984dfa1fa10), phy(4), device_name(0x500a0984dfa1fa10) [ 8.353482] scsi 1:0:1:1: enclosure logical id(0x300605b00d114820), slot(9) [ 8.353484] scsi 1:0:1:1: enclosure level(0x0000), connector name( C2 ) [ 8.353559] scsi 1:0:1:1: serial_number(021825001369 ) [ 8.353562] scsi 1:0:1:1: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.353809] scsi 1:0:1:1: Mode parameters changed [ 8.405899] scsi 1:0:1:31: Direct-Access DELL Universal Xport 0825 PQ: 0 ANSI: 5 [ 8.415243] systemd-journald[350]: Received SIGTERM from PID 1 (systemd). [ 8.423511] scsi 1:0:1:31: SSP: handle(0x0019), sas_addr(0x500a0984dfa1fa10), phy(4), device_name(0x500a0984dfa1fa10) [ 8.435416] scsi 1:0:1:31: enclosure logical id(0x300605b00d114820), slot(9) [ 8.442549] scsi 1:0:1:31: enclosure level(0x0000), connector name( C2 ) [ 8.450762] scsi 1:0:1:31: serial_number(021825001369 ) [ 8.457591] scsi 1:0:1:31: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.462895] SELinux: Disabled at runtime. [ 8.462913] SELinux: Unregistering netfilter hooks [ 8.484952] mpt3sas_cm0: detecting: handle(0x0017), sas_address(0x500a0984da0f9b24), phy(8) [ 8.494335] mpt3sas_cm0: REPORT_LUNS: handle(0x0017), retries(0) [ 8.502504] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0017), lun(0) [ 8.509728] scsi 1:0:2:0: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.518691] scsi 1:0:2:0: SSP: handle(0x0017), sas_addr(0x500a0984da0f9b24), phy(8), device_name(0x500a0984da0f9b24) [ 8.530485] scsi 1:0:2:0: enclosure logical id(0x300605b00d114820), slot(5) [ 8.537532] scsi 1:0:2:0: enclosure level(0x0000), connector name( C1 ) [ 8.545634] scsi 1:0:2:0: serial_number(021812047179 ) [ 8.552408] scsi 1:0:2:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.574660] scsi 1:0:2:1: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.583421] scsi 1:0:2:1: SSP: handle(0x0017), sas_addr(0x500a0984da0f9b24), phy(8), device_name(0x500a0984da0f9b24) [ 8.595210] scsi 1:0:2:1: enclosure logical id(0x300605b00d114820), slot(5) [ 8.602256] scsi 1:0:2:1: enclosure level(0x0000), connector name( C1 ) [ 8.610352] scsi 1:0:2:1: serial_number(021812047179 ) [ 8.617122] scsi 1:0:2:1: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.634696] type=1404 audit(1594225439.133:2): selinux=0 auid=4294967295 ses=4294967295 [ 8.644968] scsi 1:0:2:2: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.653355] scsi 1:0:2:2: SSP: handle(0x0017), sas_addr(0x500a0984da0f9b24), phy(8), device_name(0x500a0984da0f9b24) [ 8.665151] scsi 1:0:2:2: enclosure logical id(0x300605b00d114820), slot(5) [ 8.672199] scsi 1:0:2:2: enclosure level(0x0000), connector name( C1 ) [ 8.680301] scsi 1:0:2:2: serial_number(021812047179 ) [ 8.687074] scsi 1:0:2:2: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.710963] scsi 1:0:2:31: Direct-Access DELL Universal Xport 0825 PQ: 0 ANSI: 5 [ 8.719537] scsi 1:0:2:31: SSP: handle(0x0017), sas_addr(0x500a0984da0f9b24), phy(8), device_name(0x500a0984da0f9b24) [ 8.731437] scsi 1:0:2:31: enclosure logical id(0x300605b00d114820), slot(5) [ 8.738570] scsi 1:0:2:31: enclosure level(0x0000), connector name( C1 ) [ 8.746746] scsi 1:0:2:31: serial_number(021812047179 ) [ 8.753620] scsi 1:0:2:31: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.764105] ip_tables: (C) 2000-2006 Netfilter Core Team [ 8.771389] systemd[1]: Inserted module 'ip_tables' [ 8.790955] mpt3sas_cm0: detecting: handle(0x001a), sas_address(0x500a0984db2fa910), phy(12) [ 8.799412] mpt3sas_cm0: REPORT_LUNS: handle(0x001a), retries(0) [ 8.806352] mpt3sas_cm0: TEST_UNIT_READY: handle(0x001a), lun(0) [ 8.812866] scsi 1:0:3:0: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.821071] scsi 1:0:3:0: SSP: handle(0x001a), sas_addr(0x500a0984db2fa910), phy(12), device_name(0x500a0984db2fa910) [ 8.831672] scsi 1:0:3:0: enclosure logical id(0x300605b00d114820), slot(1) [ 8.838716] scsi 1:0:3:0: enclosure level(0x0000), connector name( C0 ) [ 8.845456] scsi 1:0:3:0: serial_number(021815000354 ) [ 8.850865] scsi 1:0:3:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.876667] scsi 1:0:3:1: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.884653] EXT4-fs (sda2): re-mounted. Opts: (null) [ 8.892394] scsi 1:0:3:1: SSP: handle(0x001a), sas_addr(0x500a0984db2fa910), phy(12), device_name(0x500a0984db2fa910) [ 8.904267] scsi 1:0:3:1: enclosure logical id(0x300605b00d114820), slot(1) [ 8.904268] scsi 1:0:3:1: enclosure level(0x0000), connector name( C0 ) [ 8.904289] scsi 1:0:3:1: serial_number(021815000354 ) [ 8.904292] scsi 1:0:3:1: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.904529] scsi 1:0:3:1: Mode parameters changed [ 8.950804] systemd-journald[4862]: Received request to flush runtime journal from PID 1 [ 8.952913] scsi 1:0:3:2: Direct-Access DELL MD34xx 0825 PQ: 0 ANSI: 5 [ 8.952994] scsi 1:0:3:2: SSP: handle(0x001a), sas_addr(0x500a0984db2fa910), phy(12), device_name(0x500a0984db2fa910) [ 8.952995] scsi 1:0:3:2: enclosure logical id(0x300605b00d114820), slot(1) [ 8.952997] scsi 1:0:3:2: enclosure level(0x0000), connector name( C0 ) [ 8.952998] scsi 1:0:3:2: serial_number(021815000354 ) [ 8.952999] scsi 1:0:3:2: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 8.953227] scsi 1:0:3:2: Mode parameters changed [ 9.029901] scsi 1:0:3:31: Direct-Access DELL Universal Xport 0825 PQ: 0 ANSI: 5 [ 9.039572] scsi 1:0:3:31: SSP: handle(0x001a), sas_addr(0x500a0984db2fa910), phy(12), device_name(0x500a0984db2fa910) [ 9.051555] scsi 1:0:3:31: enclosure logical id(0x300605b00d114820), slot(1) [ 9.060073] scsi 1:0:3:31: enclosure level(0x0000), connector name( C0 ) [ 9.066896] scsi 1:0:3:31: serial_number(021815000354 ) [ 9.073731] scsi 1:0:3:31: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1) [ 9.084584] device-mapper: uevent: version 1.0.3 [ 9.090293] device-mapper: ioctl: 4.37.1-ioctl (2018-04-03) initialised: dm-devel@redhat.com [ 9.100549] sd 0:2:0:0: Attached scsi generic sg0 type 0 [ 9.117305] ACPI Error: No handler for Region [SYSI] (ffff8ecca9e99b88) [IPMI] (20130517/evregion-162) [ 9.129895] ACPI Error: Region IPMI (ID=7) has no handler (20130517/exfldio-305) [ 9.141558] ACPI Error: Method parse/execution failed [\_SB_.PMI0._GHL] (Node ffff8ecca9e7a5a0), AE_NOT_EXIST (20130517/psparse-536) [ 9.161572] mpt3sas_cm0: detecting: handle(0x0011), sas_address(0x300705b00deb4820), phy(16) [ 9.161579] mpt3sas_cm0: REPORT_LUNS: handle(0x0011), retries(0) [ 9.161603] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0011), lun(0) [ 9.163341] scsi 1:0:4:0: Enclosure LSI VirtualSES 03 PQ: 0 ANSI: 7 [ 9.163369] scsi 1:0:4:0: set ignore_delay_remove for handle(0x0011) [ 9.163373] scsi 1:0:4:0: SES: handle(0x0011), sas_addr(0x300705b00deb4820), phy(16), device_name(0x300705b00deb4820) [ 9.163375] scsi 1:0:4:0: enclosure logical id(0x300605b00d114820), slot(16) [ 9.163377] scsi 1:0:4:0: enclosure level(0x0000), connector name( C3 ) [ 9.163378] scsi 1:0:4:0: serial_number(300605B00D114820) [ 9.163381] scsi 1:0:4:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(8), cmd_que(0) [ 9.163403] mpt3sas_cm0: log_info(0x31200206): originator(PL), code(0x20), sub_code(0x0206) [ 9.149281] ACPI Error: Method parse/execution failed [\_SB_.PMI0._PMC] (Node ffff8ecca9e7a500), AE_NOT_EXIST (20130517/psparse-536) [ 9.234725] ACPI Exception: AE_NOT_EXIST, [ 9.234725] Evaluating _PMC [ 9.234727] (20130517/power_meter-753) [ 9.276439] cryptd: max_cpu_qlen set to 1000 [ 9.277839] ipmi message handler version 39.2 [ 9.278835] piix4_smbus 0000:00:14.0: SMBus Host Controller at 0xb00, revision 0 [ 9.278837] piix4_smbus 0000:00:14.0: Using register 0x2e for SMBus port selection [ 9.315322] input: PC Speaker as /devices/platform/pcspkr/input/input2 [ 9.324457] ipmi device interface [ 9.324806] ccp 0000:02:00.2: 3 command queues available [ 9.324861] ccp 0000:02:00.2: irq 235 for MSI/MSI-X [ 9.324885] ccp 0000:02:00.2: irq 236 for MSI/MSI-X [ 9.324958] ccp 0000:02:00.2: Queue 2 can access 4 LSB regions [ 9.324960] ccp 0000:02:00.2: Queue 3 can access 4 LSB regions [ 9.324962] ccp 0000:02:00.2: Queue 4 can access 4 LSB regions [ 9.324964] ccp 0000:02:00.2: Queue 0 gets LSB 4 [ 9.324965] ccp 0000:02:00.2: Queue 1 gets LSB 5 [ 9.324967] ccp 0000:02:00.2: Queue 2 gets LSB 6 [ 9.325581] ccp 0000:02:00.2: enabled [ 9.326041] ccp 0000:03:00.1: 5 command queues available [ 9.326093] ccp 0000:03:00.1: irq 238 for MSI/MSI-X [ 9.326137] ccp 0000:03:00.1: Queue 0 can access 7 LSB regions [ 9.326140] ccp 0000:03:00.1: Queue 1 can access 7 LSB regions [ 9.326141] ccp 0000:03:00.1: Queue 2 can access 7 LSB regions [ 9.326144] ccp 0000:03:00.1: Queue 3 can access 7 LSB regions [ 9.326146] ccp 0000:03:00.1: Queue 4 can access 7 LSB regions [ 9.326148] ccp 0000:03:00.1: Queue 0 gets LSB 1 [ 9.326149] ccp 0000:03:00.1: Queue 1 gets LSB 2 [ 9.326150] ccp 0000:03:00.1: Queue 2 gets LSB 3 [ 9.326152] ccp 0000:03:00.1: Queue 3 gets LSB 4 [ 9.326153] ccp 0000:03:00.1: Queue 4 gets LSB 5 [ 9.326846] ccp 0000:03:00.1: enabled [ 9.327096] ccp 0000:41:00.2: 3 command queues available [ 9.327147] ccp 0000:41:00.2: irq 240 for MSI/MSI-X [ 9.327170] ccp 0000:41:00.2: irq 241 for MSI/MSI-X [ 9.327228] ccp 0000:41:00.2: Queue 2 can access 4 LSB regions [ 9.327230] ccp 0000:41:00.2: Queue 3 can access 4 LSB regions [ 9.327232] ccp 0000:41:00.2: Queue 4 can access 4 LSB regions [ 9.327233] ccp 0000:41:00.2: Queue 0 gets LSB 4 [ 9.327234] ccp 0000:41:00.2: Queue 1 gets LSB 5 [ 9.327235] ccp 0000:41:00.2: Queue 2 gets LSB 6 [ 9.327683] ccp 0000:41:00.2: enabled [ 9.327833] ccp 0000:42:00.1: 5 command queues available [ 9.327873] ccp 0000:42:00.1: irq 243 for MSI/MSI-X [ 9.327946] ccp 0000:42:00.1: Queue 0 can access 7 LSB regions [ 9.327948] ccp 0000:42:00.1: Queue 1 can access 7 LSB regions [ 9.327951] ccp 0000:42:00.1: Queue 2 can access 7 LSB regions [ 9.327953] ccp 0000:42:00.1: Queue 3 can access 7 LSB regions [ 9.327955] ccp 0000:42:00.1: Queue 4 can access 7 LSB regions [ 9.327957] ccp 0000:42:00.1: Queue 0 gets LSB 1 [ 9.327958] ccp 0000:42:00.1: Queue 1 gets LSB 2 [ 9.327960] ccp 0000:42:00.1: Queue 2 gets LSB 3 [ 9.327961] ccp 0000:42:00.1: Queue 3 gets LSB 4 [ 9.327962] ccp 0000:42:00.1: Queue 4 gets LSB 5 [ 9.329666] ccp 0000:42:00.1: enabled [ 9.329887] ccp 0000:85:00.2: 3 command queues available [ 9.329944] ccp 0000:85:00.2: irq 245 for MSI/MSI-X [ 9.329968] ccp 0000:85:00.2: irq 246 for MSI/MSI-X [ 9.330032] ccp 0000:85:00.2: Queue 2 can access 4 LSB regions [ 9.330034] ccp 0000:85:00.2: Queue 3 can access 4 LSB regions [ 9.330036] ccp 0000:85:00.2: Queue 4 can access 4 LSB regions [ 9.330038] ccp 0000:85:00.2: Queue 0 gets LSB 4 [ 9.330040] ccp 0000:85:00.2: Queue 1 gets LSB 5 [ 9.330041] ccp 0000:85:00.2: Queue 2 gets LSB 6 [ 9.330741] ccp 0000:85:00.2: enabled [ 9.330894] ccp 0000:86:00.1: 5 command queues available [ 9.330941] ccp 0000:86:00.1: irq 248 for MSI/MSI-X [ 9.330974] ccp 0000:86:00.1: Queue 0 can access 7 LSB regions [ 9.330976] ccp 0000:86:00.1: Queue 1 can access 7 LSB regions [ 9.330978] ccp 0000:86:00.1: Queue 2 can access 7 LSB regions [ 9.330980] ccp 0000:86:00.1: Queue 3 can access 7 LSB regions [ 9.330982] ccp 0000:86:00.1: Queue 4 can access 7 LSB regions [ 9.330983] ccp 0000:86:00.1: Queue 0 gets LSB 1 [ 9.330985] ccp 0000:86:00.1: Queue 1 gets LSB 2 [ 9.330986] ccp 0000:86:00.1: Queue 2 gets LSB 3 [ 9.330987] ccp 0000:86:00.1: Queue 3 gets LSB 4 [ 9.330988] ccp 0000:86:00.1: Queue 4 gets LSB 5 [ 9.333013] ccp 0000:86:00.1: enabled [ 9.333305] ccp 0000:c2:00.2: 3 command queues available [ 9.333362] ccp 0000:c2:00.2: irq 250 for MSI/MSI-X [ 9.333389] ccp 0000:c2:00.2: irq 251 for MSI/MSI-X [ 9.333443] ccp 0000:c2:00.2: Queue 2 can access 4 LSB regions [ 9.333445] ccp 0000:c2:00.2: Queue 3 can access 4 LSB regions [ 9.333448] ccp 0000:c2:00.2: Queue 4 can access 4 LSB regions [ 9.333450] ccp 0000:c2:00.2: Queue 0 gets LSB 4 [ 9.333451] ccp 0000:c2:00.2: Queue 1 gets LSB 5 [ 9.333452] ccp 0000:c2:00.2: Queue 2 gets LSB 6 [ 9.333954] ccp 0000:c2:00.2: enabled [ 9.334076] ccp 0000:c3:00.1: 5 command queues available [ 9.334127] ccp 0000:c3:00.1: irq 253 for MSI/MSI-X [ 9.334164] ccp 0000:c3:00.1: Queue 0 can access 7 LSB regions [ 9.334166] ccp 0000:c3:00.1: Queue 1 can access 7 LSB regions [ 9.334168] ccp 0000:c3:00.1: Queue 2 can access 7 LSB regions [ 9.334170] ccp 0000:c3:00.1: Queue 3 can access 7 LSB regions [ 9.334172] ccp 0000:c3:00.1: Queue 4 can access 7 LSB regions [ 9.334174] ccp 0000:c3:00.1: Queue 0 gets LSB 1 [ 9.334175] ccp 0000:c3:00.1: Queue 1 gets LSB 2 [ 9.334176] ccp 0000:c3:00.1: Queue 2 gets LSB 3 [ 9.334177] ccp 0000:c3:00.1: Queue 3 gets LSB 4 [ 9.334178] ccp 0000:c3:00.1: Queue 4 gets LSB 5 [ 9.334884] ccp 0000:c3:00.1: enabled [ 9.847599] IPMI System Interface driver [ 9.847732] dcdbas dcdbas: Dell Systems Management Base Driver (version 5.6.0-3.3) [ 9.853157] AVX2 version of gcm_enc/dec engaged. [ 9.853158] AES CTR mode by8 optimization enabled [ 9.874095] ipmi_si dmi-ipmi-si.0: ipmi_platform: probing via SMBIOS [ 9.874098] ipmi_si: SMBIOS: io 0xca8 regsize 1 spacing 4 irq 10 [ 9.874099] ipmi_si: Adding SMBIOS-specified kcs state machine [ 9.874130] ipmi_si IPI0001:00: ipmi_platform: probing via ACPI [ 9.874155] ipmi_si IPI0001:00: [io 0x0ca8] regsize 1 spacing 4 irq 10 [ 9.874156] ipmi_si dmi-ipmi-si.0: Removing SMBIOS-specified kcs state machine in favor of ACPI [ 9.874159] ipmi_si: Adding ACPI-specified kcs state machine [ 9.874248] ipmi_si: Trying ACPI-specified kcs state machine at i/o address 0xca8, slave address 0x20, irq 10 [ 9.892855] alg: No test for __gcm-aes-aesni (__driver-gcm-aes-aesni) [ 9.892937] alg: No test for __generic-gcm-aes-aesni (__driver-generic-gcm-aes-aesni) [ 9.897722] mpt3sas_cm0: port enable: SUCCESS [ 9.898231] scsi 1:0:0:0: rdac: LUN 0 (IOSHIP) (unowned) [ 9.898412] sd 1:0:0:0: Attached scsi generic sg1 type 0 [ 9.898459] sd 1:0:0:0: [sdb] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.898912] scsi 1:0:0:1: rdac: LUN 1 (IOSHIP) (owned) [ 9.899000] sd 1:0:0:0: [sdb] Write Protect is off [ 9.899002] sd 1:0:0:0: [sdb] Mode Sense: 83 00 10 08 [ 9.899147] sd 1:0:0:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.899225] sd 1:0:0:1: Attached scsi generic sg2 type 0 [ 9.899327] sd 1:0:0:1: [sdc] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.899743] sd 1:0:0:1: [sdc] Write Protect is off [ 9.899744] sd 1:0:0:1: [sdc] Mode Sense: 83 00 10 08 [ 9.899788] scsi 1:0:0:31: Attached scsi generic sg3 type 0 [ 9.899900] sd 1:0:0:1: [sdc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.900455] scsi 1:0:1:0: rdac: LUN 0 (IOSHIP) (owned) [ 9.900730] sd 1:0:1:0: [sdd] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.900766] sd 1:0:1:0: Attached scsi generic sg4 type 0 [ 9.901308] sd 1:0:1:0: [sdd] Write Protect is off [ 9.901309] sd 1:0:1:0: [sdd] Mode Sense: 83 00 10 08 [ 9.901587] sd 1:0:1:0: [sdd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.902063] scsi 1:0:1:1: rdac: LUN 1 (IOSHIP) (unowned) [ 9.902602] sd 1:0:1:1: [sde] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.902695] sd 1:0:1:1: Attached scsi generic sg5 type 0 [ 9.903162] sd 1:0:1:1: [sde] Write Protect is off [ 9.903164] sd 1:0:1:1: [sde] Mode Sense: 83 00 10 08 [ 9.903390] sd 1:0:1:1: [sde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.903426] scsi 1:0:1:31: Attached scsi generic sg6 type 0 [ 9.904564] scsi 1:0:2:0: rdac: LUN 0 (IOSHIP) (unowned) [ 9.905423] sd 1:0:2:0: Attached scsi generic sg7 type 0 [ 9.905812] sd 1:0:2:0: [sdf] 926167040 512-byte logical blocks: (474 GB/441 GiB) [ 9.905814] sd 1:0:2:0: [sdf] 4096-byte physical blocks [ 9.906167] sd 1:0:0:0: [sdb] Attached SCSI disk [ 9.906183] sd 1:0:0:1: [sdc] Attached SCSI disk [ 9.906199] sd 1:0:1:0: [sdd] Attached SCSI disk [ 9.906549] sd 1:0:2:0: [sdf] Write Protect is off [ 9.906550] sd 1:0:2:0: [sdf] Mode Sense: 83 00 10 08 [ 9.906580] scsi 1:0:2:1: rdac: LUN 1 (IOSHIP) (owned) [ 9.906756] sd 1:0:2:0: [sdf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.907641] sd 1:0:2:1: [sdg] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.908797] sd 1:0:2:1: [sdg] Write Protect is off [ 9.908799] sd 1:0:2:1: [sdg] Mode Sense: 83 00 10 08 [ 9.908820] sd 1:0:1:1: [sde] Attached SCSI disk [ 9.908852] sd 1:0:2:1: Attached scsi generic sg8 type 0 [ 9.909082] sd 1:0:2:1: [sdg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.909996] scsi 1:0:2:2: rdac: LUN 2 (IOSHIP) (unowned) [ 9.910599] sd 1:0:2:2: Attached scsi generic sg9 type 0 [ 9.910668] sd 1:0:2:2: [sdh] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.911648] sd 1:0:2:2: [sdh] Write Protect is off [ 9.911649] scsi 1:0:2:31: Attached scsi generic sg10 type 0 [ 9.911650] sd 1:0:2:2: [sdh] Mode Sense: 83 00 10 08 [ 9.911948] sd 1:0:2:2: [sdh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.913037] sd 1:0:2:0: [sdf] Attached SCSI disk [ 9.913406] scsi 1:0:3:0: rdac: LUN 0 (IOSHIP) (owned) [ 9.914265] sd 1:0:2:1: [sdg] Attached SCSI disk [ 9.914396] sd 1:0:3:0: [sdi] 926167040 512-byte logical blocks: (474 GB/441 GiB) [ 9.914397] sd 1:0:3:0: [sdi] 4096-byte physical blocks [ 9.914733] sd 1:0:3:0: Attached scsi generic sg11 type 0 [ 9.914786] sd 1:0:3:0: [sdi] Write Protect is off [ 9.914788] sd 1:0:3:0: [sdi] Mode Sense: 83 00 10 08 [ 9.914927] sd 1:0:3:0: [sdi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.916128] scsi 1:0:3:1: rdac: LUN 1 (IOSHIP) (unowned) [ 9.916943] sd 1:0:3:1: Attached scsi generic sg12 type 0 [ 9.917505] sd 1:0:3:1: [sdj] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.917650] sd 1:0:2:2: [sdh] Attached SCSI disk [ 9.917987] sd 1:0:3:1: [sdj] Write Protect is off [ 9.917988] sd 1:0:3:1: [sdj] Mode Sense: 83 00 10 08 [ 9.918223] scsi 1:0:3:2: rdac: LUN 2 (IOSHIP) (owned) [ 9.918303] sd 1:0:3:1: [sdj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.918792] sd 1:0:3:2: [sdk] 37449707520 512-byte logical blocks: (19.1 TB/17.4 TiB) [ 9.919041] sd 1:0:3:0: [sdi] Attached SCSI disk [ 9.919057] sd 1:0:3:2: Attached scsi generic sg13 type 0 [ 9.920314] sd 1:0:3:2: [sdk] Write Protect is off [ 9.920316] sd 1:0:3:2: [sdk] Mode Sense: 83 00 10 08 [ 9.920511] sd 1:0:3:2: [sdk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 9.922193] scsi 1:0:3:31: Attached scsi generic sg14 type 0 [ 9.925102] scsi 1:0:4:0: Attached scsi generic sg15 type 13 [ 9.925412] sd 1:0:3:1: [sdj] Attached SCSI disk [ 9.925976] sd 1:0:3:2: [sdk] Attached SCSI disk [ 9.950739] ipmi_si IPI0001:00: The BMC does not support setting the recv irq bit, compensating, but the BMC needs to be fixed. [ 9.962787] ipmi_si IPI0001:00: Using irq 10 [ 9.986681] ipmi_si IPI0001:00: Found new BMC (man_id: 0x0002a2, prod_id: 0x0100, dev_id: 0x20) [ 10.043952] sd 1:0:0:0: Embedded Enclosure Device [ 10.046274] sd 1:0:0:1: Embedded Enclosure Device [ 10.048402] scsi 1:0:0:31: Embedded Enclosure Device [ 10.050520] sd 1:0:1:0: Embedded Enclosure Device [ 10.052704] sd 1:0:1:1: Embedded Enclosure Device [ 10.054780] scsi 1:0:1:31: Embedded Enclosure Device [ 10.056865] sd 1:0:2:0: Embedded Enclosure Device [ 10.057708] ipmi_si IPI0001:00: IPMI kcs interface initialized [ 10.059323] sd 1:0:2:1: Embedded Enclosure Device [ 10.061662] sd 1:0:2:2: Embedded Enclosure Device [ 10.063982] scsi 1:0:2:31: Embedded Enclosure Device [ 10.066275] sd 1:0:3:0: Embedded Enclosure Device [ 10.068429] sd 1:0:3:1: Embedded Enclosure Device [ 10.070461] sd 1:0:3:2: Embedded Enclosure Device [ 10.072492] scsi 1:0:3:31: Embedded Enclosure Device [ 10.075158] ses 1:0:4:0: Attached Enclosure device [ 10.395123] kvm: Nested Paging enabled [ 10.468229] MCE: In-kernel MCE decoding enabled. [ 10.505659] AMD64 EDAC driver v3.4.0 [ 10.509262] EDAC amd64: DRAM ECC enabled. [ 10.513284] EDAC amd64: F17h detected (node 0). [ 10.517868] EDAC MC: UMC0 chip selects: [ 10.517870] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.522583] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.527297] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.532004] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.536711] EDAC MC: UMC1 chip selects: [ 10.536715] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.541425] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.546140] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.546142] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.546143] EDAC amd64: using x8 syndromes. [ 10.546147] EDAC amd64: MCT channel count: 2 [ 10.546313] EDAC MC0: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:18.3 [ 10.546319] EDAC amd64: DRAM ECC enabled. [ 10.546320] EDAC amd64: F17h detected (node 1). [ 10.546361] EDAC MC: UMC0 chip selects: [ 10.546362] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.546363] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.546364] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.546364] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.546367] EDAC MC: UMC1 chip selects: [ 10.546367] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.546368] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.546369] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.546369] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.546370] EDAC amd64: using x8 syndromes. [ 10.546370] EDAC amd64: MCT channel count: 2 [ 10.546524] EDAC MC1: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:19.3 [ 10.546529] EDAC amd64: DRAM ECC enabled. [ 10.546530] EDAC amd64: F17h detected (node 2). [ 10.546567] EDAC MC: UMC0 chip selects: [ 10.546568] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.546568] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.546569] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.546569] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.546571] EDAC MC: UMC1 chip selects: [ 10.546572] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.546573] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.546573] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.546574] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.546574] EDAC amd64: using x8 syndromes. [ 10.546574] EDAC amd64: MCT channel count: 2 [ 10.547533] EDAC MC2: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1a.3 [ 10.547539] EDAC amd64: DRAM ECC enabled. [ 10.547540] EDAC amd64: F17h detected (node 3). [ 10.547584] EDAC MC: UMC0 chip selects: [ 10.547585] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.547586] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.547586] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.547587] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.547590] EDAC MC: UMC1 chip selects: [ 10.547590] EDAC amd64: MC: 0: 0MB 1: 0MB [ 10.547591] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 10.547591] EDAC amd64: MC: 4: 0MB 5: 0MB [ 10.547592] EDAC amd64: MC: 6: 0MB 7: 0MB [ 10.547592] EDAC amd64: using x8 syndromes. [ 10.547593] EDAC amd64: MCT channel count: 2 [ 10.548586] EDAC MC3: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1b.3 [ 10.548668] EDAC PCI0: Giving out device to module 'amd64_edac' controller 'EDAC PCI controller': DEV '0000:00:18.0' (POLLED) [ 38.039947] device-mapper: multipath round-robin: version 1.2.0 loaded [ 48.396172] Adding 4194300k swap on /dev/sda3. Priority:-2 extents:1 across:4194300k FS [ 48.409836] FAT-fs (sda1): Volume was not properly unmounted. Some data may be corrupt. Please run fsck. [ 48.446975] type=1305 audit(1594225478.944:3): audit_pid=10250 old=0 auid=4294967295 ses=4294967295 res=1 [ 48.467472] RPC: Registered named UNIX socket transport module. [ 48.474634] RPC: Registered udp transport module. [ 48.480728] RPC: Registered tcp transport module. [ 48.486820] RPC: Registered tcp NFSv4.1 backchannel transport module. [ 49.095028] mlx5_core 0000:01:00.0: slow_pci_heuristic:5575:(pid 10541): Max link speed = 100000, PCI BW = 126016 [ 49.105352] mlx5_core 0000:01:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(256) RxCqeCmprss(0) [ 49.113627] mlx5_core 0000:01:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(256) RxCqeCmprss(0) [ 49.545466] tg3 0000:81:00.0: irq 254 for MSI/MSI-X [ 49.545484] tg3 0000:81:00.0: irq 255 for MSI/MSI-X [ 49.545496] tg3 0000:81:00.0: irq 256 for MSI/MSI-X [ 49.545507] tg3 0000:81:00.0: irq 257 for MSI/MSI-X [ 49.545517] tg3 0000:81:00.0: irq 258 for MSI/MSI-X [ 49.671582] IPv6: ADDRCONF(NETDEV_UP): em1: link is not ready [ 53.244743] tg3 0000:81:00.0 em1: Link is up at 1000 Mbps, full duplex [ 53.251276] tg3 0000:81:00.0 em1: Flow control is on for TX and on for RX [ 53.258068] tg3 0000:81:00.0 em1: EEE is enabled [ 53.262702] IPv6: ADDRCONF(NETDEV_CHANGE): em1: link becomes ready [ 54.033558] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready [ 54.333212] IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready [ 58.169131] FS-Cache: Loaded [ 58.199162] FS-Cache: Netfs 'nfs' registered for caching [ 58.208011] Key type dns_resolver registered [ 58.236211] NFS: Registering the id_resolver key type [ 58.241677] Key type id_resolver registered [ 58.247243] Key type id_legacy registered [17572.236090] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [17572.243666] alg: No test for adler32 (adler32-zlib) [17573.043240] Lustre: Lustre: Build Version: 2.12.5_RC1 [17573.149115] LNet: 23888:0:(config.c:1642:lnet_inet_enumerate()) lnet: Ignoring interface em2: it's down [17573.158861] LNet: Using FastReg for registration [17573.175000] LNet: Added LNI 10.0.10.54@o2ib7 [8/256/0/180] [17574.148844] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.224@o2ib7: 17573 seconds [17575.148860] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.224@o2ib7: 17574 seconds [17575.159377] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [17579.373074] LDISKFS-fs (dm-0): file extents enabled, maximum tree depth=5 [17581.187361] LDISKFS-fs (dm-0): mounted filesystem with ordered data mode. Opts: user_xattr,errors=remount-ro,acl,no_mbcache,nodelalloc [17582.049554] LustreError: 11-0: fir-MDT0000-osp-MDT0003: operation mds_connect to node 10.0.10.51@o2ib7 failed: rc = -114 [17582.208861] Lustre: fir-MDT0003: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-900 [17582.223109] Lustre: fir-MDD0003: changelog on [17582.227572] Lustre: 24020:0:(mdd_device.c:545:mdd_changelog_llog_init()) fir-MDD0003 : orphan changelog records found, starting from index 0 to index 17291100569, being cleared now [17582.250579] Lustre: fir-MDT0003: in recovery but waiting for the first client to connect [17583.186297] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 10.0.10.110@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17583.203763] Lustre: fir-MDT0003: Connection restored to 10.0.10.110@o2ib7 (at 10.0.10.110@o2ib7) [17584.796398] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.112@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17584.796482] Lustre: fir-MDT0003: Connection restored to 10.0.10.112@o2ib7 (at 10.0.10.112@o2ib7) [17584.796484] Lustre: Skipped 1 previous similar message [17584.827694] LustreError: Skipped 9 previous similar messages [17585.149135] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.224@o2ib7: 0 seconds [17585.159298] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [17586.850184] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 10.0.10.104@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17586.867550] LustreError: Skipped 6 previous similar messages [17588.081279] Lustre: fir-MDT0003: Connection restored to 10.0.10.113@o2ib7 (at 10.0.10.113@o2ib7) [17590.009304] LustreError: 137-5: fir-MDT0001_UUID: not available for connect from 10.0.10.114@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17590.026670] LustreError: Skipped 23 previous similar messages [17591.186251] Lustre: fir-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [17591.195049] Lustre: Skipped 1 previous similar message [17594.970417] LustreError: 137-5: fir-MDT0001_UUID: not available for connect from 10.0.10.109@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17594.987785] LustreError: Skipped 28 previous similar messages [17599.067384] Lustre: fir-MDT0003: Connection restored to 10.0.10.103@o2ib7 (at 10.0.10.103@o2ib7) [17599.076175] Lustre: Skipped 11 previous similar messages [17604.494184] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17604.511472] LustreError: Skipped 10 previous similar messages [17607.270007] LustreError: 11-0: fir-OST0000-osc-MDT0003: operation ost_connect to node 10.0.10.101@o2ib7 failed: rc = -16 [17607.280874] LustreError: Skipped 193 previous similar messages [17607.481856] Lustre: fir-MDT0003: Will be in recovery for at least 2:30, or until 2 clients reconnect [17607.491066] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.53@o2ib7) [17607.498291] Lustre: Skipped 2 previous similar messages [17624.156024] LustreError: 137-5: fir-MDT0001_UUID: not available for connect from 10.0.10.103@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [17624.173416] LustreError: Skipped 40 previous similar messages [17632.294788] LustreError: 11-0: fir-OST0000-osc-MDT0003: operation ost_connect to node 10.0.10.101@o2ib7 failed: rc = -16 [17632.305656] LustreError: Skipped 95 previous similar messages [17633.363611] Lustre: fir-MDT0003: Connection restored to 10.0.10.110@o2ib7 (at 10.0.10.110@o2ib7) [17633.372406] Lustre: Skipped 26 previous similar messages [17657.383230] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. [17657.383505] LustreError: 11-0: fir-OST0000-osc-MDT0003: operation ost_connect to node 10.0.10.101@o2ib7 failed: rc = -16 [17657.383507] LustreError: Skipped 95 previous similar messages [17657.416091] LustreError: Skipped 32 previous similar messages [17666.452199] Lustre: fir-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [17666.460990] Lustre: Skipped 38 previous similar messages [17682.472039] LustreError: 11-0: fir-OST0000-osc-MDT0003: operation ost_connect to node 10.0.10.101@o2ib7 failed: rc = -16 [17682.482905] LustreError: Skipped 95 previous similar messages [17707.560616] LustreError: 11-0: fir-OST0000-osc-MDT0003: operation ost_connect to node 10.0.10.101@o2ib7 failed: rc = -16 [17707.571493] LustreError: Skipped 95 previous similar messages [17732.649373] LustreError: 11-0: fir-OST0001-osc-MDT0003: operation ost_connect to node 10.0.10.102@o2ib7 failed: rc = -16 [17732.660253] LustreError: Skipped 95 previous similar messages [17756.714805] Lustre: fir-MDT0003: recovery is timed out, evict stale exports [17756.721831] Lustre: fir-MDT0003: disconnecting 2 stale clients [17756.736571] LustreError: 24478:0:(fld_handler.c:225:fld_local_lookup()) srv-fir-MDT0003: FLD cache range [0x0000001440000400-0x0000001480000400]:45:ost does not match requested flag 0: rc = -5 [17756.802402] Lustre: fir-MDT0003: Recovery over after 2:30, of 5 clients 3 recovered and 2 were evicted. [17782.826889] LustreError: 11-0: fir-OST0001-osc-MDT0003: operation ost_connect to node 10.0.10.102@o2ib7 failed: rc = -16 [17782.837756] LustreError: Skipped 183 previous similar messages [17829.716326] Lustre: fir-MDT0001-osp-MDT0003: Connection restored to 10.0.10.52@o2ib7 (at 10.0.10.52@o2ib7) [17829.725983] Lustre: Skipped 13 previous similar messages [17858.092717] LustreError: 11-0: fir-OST0000-osc-MDT0003: operation ost_connect to node 10.0.10.101@o2ib7 failed: rc = -16 [17858.103584] LustreError: Skipped 275 previous similar messages [17877.157082] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.224@o2ib7: 0 seconds [17877.167250] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [18165.855683] Lustre: fir-MDT0003: Connection restored to 8b4d6727-b5d7-4 (at 10.50.0.1@o2ib2) [18165.864127] Lustre: Skipped 2 previous similar messages [18179.165175] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [18179.175345] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 3 previous similar messages [18483.173305] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 2 seconds [18483.183472] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [18629.671270] Lustre: fir-MDT0003: haven't heard from client ac340994-7cfa-4 (at 10.51.0.13@o2ib3) in 228 seconds. I think it's dead, and I am evicting it. exp ffff8efbf5af6c00, cur 1594244060 expire 1594243910 last 1594243832 [18782.181335] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [18782.191510] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [19083.189374] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 1 seconds [19083.199546] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [19383.197394] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [19383.207567] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [19446.470254] Lustre: fir-MDT0003: Connection restored to 8b4d6727-b5d7-4 (at 10.50.0.1@o2ib2) [19446.478702] Lustre: Skipped 2 previous similar messages [19685.205474] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [19685.215645] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [19868.010411] Lustre: fir-MDT0003: Connection restored to 7fc26a72-0608-4 (at 10.50.1.4@o2ib2) [19992.213673] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [19992.223843] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [20460.582149] Lustre: fir-MDT0003: Connection restored to d52b9b9f-55a6-4 (at 10.49.29.8@o2ib1) [20460.590697] Lustre: Skipped 11 previous similar messages [20588.770385] Lustre: fir-MDT0003: Connection restored to a17eb070-0fec-4 (at 10.50.7.30@o2ib2) [20588.778916] Lustre: Skipped 355 previous similar messages [20594.229987] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [20594.240160] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 2 previous similar messages [20846.572820] Lustre: fir-MDT0003: Connection restored to 7c1929aa-1c2b-4 (at 10.50.4.9@o2ib2) [20846.581294] Lustre: Skipped 634 previous similar messages [21152.472743] Lustre: fir-MDT0003: haven't heard from client 943e27a9-1209-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebc2758400, cur 1594246583 expire 1594246433 last 1594246356 [21196.246480] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [21196.256654] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 2 previous similar messages [21459.852783] Lustre: fir-MDT0003: Connection restored to a7a03a78-2bc3-4 (at 10.50.0.62@o2ib2) [21459.861314] Lustre: Skipped 222 previous similar messages [22099.271703] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [22099.281874] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [22516.743950] Lustre: fir-MDT0003: Connection restored to 83919983-33ef-4 (at 10.49.18.29@o2ib1) [22516.752564] Lustre: Skipped 61 previous similar messages [22996.296425] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [22996.306593] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 2 previous similar messages [23366.488949] Lustre: fir-MDT0003: Connection restored to 4e78023b-3181-4 (at 10.50.14.1@o2ib2) [23366.497476] Lustre: Skipped 7 previous similar messages [23458.549115] Lustre: fir-MDT0003: haven't heard from client a2e91212-e6f8-4 (at 10.50.16.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebda1d7400, cur 1594248889 expire 1594248739 last 1594248662 [23604.312733] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [23604.322905] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 2 previous similar messages [24212.329568] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.234@o2ib7: 0 seconds [24212.339737] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [24271.195241] Lustre: fir-MDT0003: Connection restored to 0d76a3ee-dfa3-4 (at 10.50.15.2@o2ib2) [24271.203771] Lustre: Skipped 46 previous similar messages [25185.500390] Lustre: fir-MDT0003: Connection restored to fc396b19-19df-4 (at 10.49.25.31@o2ib1) [25185.509014] Lustre: Skipped 3 previous similar messages [25963.487800] Lustre: fir-MDT0003: Connection restored to af8ee50c-127d-4 (at 10.49.19.7@o2ib1) [26800.051668] Lustre: fir-MDT0003: Client e0c84f82-44e1-4 (at 10.51.1.61@o2ib3) reconnecting [26800.059956] Lustre: fir-MDT0003: Connection restored to e0c84f82-44e1-4 (at 10.51.1.61@o2ib3) [26800.068483] Lustre: Skipped 1 previous similar message [26800.985718] Lustre: fir-MDT0003: Client 4f06bd36-3fc4-4 (at 10.51.1.62@o2ib3) reconnecting [27409.639664] Lustre: fir-MDT0003: haven't heard from client 204ab77c-7197-4 (at 10.50.8.19@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf7ac9800, cur 1594252840 expire 1594252690 last 1594252613 [27769.411425] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [27769.419870] Lustre: Skipped 12 previous similar messages [28633.520006] Lustre: fir-MDT0003: Connection restored to e0fca2ac-5557-4 (at 10.51.0.66@o2ib3) [28737.673103] Lustre: fir-MDT0003: haven't heard from client 84c97553-ca12-4 (at 10.51.12.17@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe1654c00, cur 1594254168 expire 1594254018 last 1594253941 [28737.693071] Lustre: Skipped 4 previous similar messages [29267.467715] Lustre: fir-MDT0003: Connection restored to 84c97553-ca12-4 (at 10.51.12.17@o2ib3) [29267.476333] Lustre: Skipped 4 previous similar messages [29637.697641] Lustre: fir-MDT0003: haven't heard from client 8ae1dee2-1de9-4 (at 10.50.8.19@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf1393c00, cur 1594255068 expire 1594254918 last 1594254841 [30640.000093] Lustre: fir-MDT0003: Connection restored to b851889f-7978-4 (at 10.50.1.21@o2ib2) [31790.595856] Lustre: fir-MDT0003: Connection restored to 74a98878-01a2-4 (at 10.50.3.47@o2ib2) [31832.673388] Lustre: fir-MDT0003: Connection restored to 204ab77c-7197-4 (at 10.50.8.19@o2ib2) [31858.439535] Lustre: fir-MDT0003: Connection restored to 0d87bdec-3769-4 (at 10.50.10.63@o2ib2) [32554.553529] Lustre: fir-MDT0003: Connection restored to d3e3cfe2-8c50-4 (at 10.50.3.22@o2ib2) [32564.421893] Lustre: fir-MDT0003: Connection restored to e9947119-97ba-4 (at 10.50.3.1@o2ib2) [37327.677014] Lustre: fir-MDT0003: Connection restored to 7afac300-e91d-4 (at 10.50.3.37@o2ib2) [37503.873152] Lustre: fir-MDT0003: Connection restored to 3d2619c0-72f9-4 (at 10.49.18.33@o2ib1) [37565.337016] Lustre: fir-MDT0003: Connection restored to 4796b170-de3d-4 (at 10.50.2.3@o2ib2) [37785.074363] Lustre: fir-MDT0003: Connection restored to d2c49c5b-480b-4 (at 10.50.16.3@o2ib2) [38377.173759] Lustre: fir-MDT0003: Connection restored to 32b3c36e-ea5f-4 (at 10.50.7.38@o2ib2) [38377.182291] Lustre: Skipped 1 previous similar message [39786.483763] Lustre: fir-MDT0003: Connection restored to 17e72c1e-4228-4 (at 10.50.6.5@o2ib2) [43633.083322] Lustre: fir-MDT0003: haven't heard from client 4234ffc3-bd92-4 (at 10.50.8.19@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbda981000, cur 1594269063 expire 1594268913 last 1594268836 [43633.103206] Lustre: Skipped 2 previous similar messages [46114.773610] Lustre: fir-MDT0003: Connection restored to 204ab77c-7197-4 (at 10.50.8.19@o2ib2) [49299.345251] perf: interrupt took too long (2502 > 2500), lowering kernel.perf_event_max_sample_rate to 79000 [57702.502027] Lustre: fir-MDT0003: haven't heard from client 4b806782-490c-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf986d400, cur 1594283132 expire 1594282982 last 1594282905 [58076.442091] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [59822.559588] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285244/real 1594285244] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285251 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [59829.586798] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285252/real 1594285252] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285259 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59836.614008] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285259/real 1594285259] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285266 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59843.642220] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285266/real 1594285266] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285273 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59850.669426] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285273/real 1594285273] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285280 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59864.696838] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285287/real 1594285287] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285294 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59864.724007] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 1 previous similar message [59885.733464] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285308/real 1594285308] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285315 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59885.760633] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [59920.772495] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594285343/real 1594285343] req@ffff8ecb6dec1680 x1671685166319168/t0(0) o104->fir-MDT0003@10.50.7.1@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1594285350 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [59920.799667] Lustre: 26725:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [59941.568268] Lustre: fir-MDT0003: haven't heard from client 9f9bd3bb-4d2a-4 (at 10.50.7.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbde466800, cur 1594285371 expire 1594285221 last 1594285144 [59941.588103] LustreError: 26725:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.7.1@o2ib2) failed to reply to blocking AST (req@ffff8ecb6dec1680 x1671685166319168 status 0 rc -5), evict it ns: mdt-fir-MDT0003_UUID lock: ffff8eeab6f1f500/0xd0d86171be96059e lrc: 4/0,0 mode: CR/CR res: [0x2800419f6:0x9:0x0].0x0 bits 0x9/0x0 rrc: 3 type: IBT flags: 0x60200400000020 nid: 10.50.7.1@o2ib2 remote: 0xb22546e617ee5db2 expref: 16570 pid: 24075 timeout: 60082 lvb_type: 0 [59941.630274] LustreError: 138-a: fir-MDT0003: A client on nid 10.50.7.1@o2ib2 was evicted due to a lock blocking callback time out: rc -5 [59941.646795] LustreError: 24064:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 0s: evicting client at 10.50.7.1@o2ib2 ns: mdt-fir-MDT0003_UUID lock: ffff8edb13a3d7c0/0xd0d86171be956ae8 lrc: 3/0,0 mode: PR/PR res: [0x2800419f6:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT flags: 0x60200400000020 nid: 10.50.7.1@o2ib2 remote: 0xb22546e617ee5d8f expref: 13936 pid: 26887 timeout: 0 lvb_type: 0 [59942.893208] LustreError: 26761:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.7.1@o2ib2) returned error from blocking AST (req@ffff8efb0829f980 x1671685167188224 status -107 rc -107), evict it ns: mdt-fir-MDT0003_UUID lock: ffff8ecb49f4d580/0xd0d86171be956b82 lrc: 4/0,0 mode: PR/PR res: [0x28003b086:0x2e7a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT flags: 0x60200400000020 nid: 10.50.7.1@o2ib2 remote: 0xb22546e617ee5da4 expref: 3622 pid: 26737 timeout: 60090 lvb_type: 0 [59942.936155] LustreError: 26761:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) Skipped 1 previous similar message [59942.946162] LustreError: 138-a: fir-MDT0003: A client on nid 10.50.7.1@o2ib2 was evicted due to a lock blocking callback time out: rc -107 [59942.958592] LustreError: Skipped 1 previous similar message [59942.964194] LustreError: 24064:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 0s: evicting client at 10.50.7.1@o2ib2 ns: mdt-fir-MDT0003_UUID lock: ffff8ecb49f4d580/0xd0d86171be956b82 lrc: 3/0,0 mode: PR/PR res: [0x28003b086:0x2e7a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT flags: 0x60200400000020 nid: 10.50.7.1@o2ib2 remote: 0xb22546e617ee5da4 expref: 3271 pid: 26737 timeout: 0 lvb_type: 0 [59949.693702] Lustre: fir-MDT0003: Connection restored to 9f9bd3bb-4d2a-4 (at 10.50.7.1@o2ib2) [60267.578107] Lustre: fir-MDT0003: haven't heard from client 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9cd2c00, cur 1594285697 expire 1594285547 last 1594285470 [63481.671691] Lustre: fir-MDT0003: haven't heard from client 08be63cd-e3e2-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf5729000, cur 1594288911 expire 1594288761 last 1594288684 [63556.279844] Lustre: fir-MDT0003: Connection restored to 08be63cd-e3e2-4 (at 10.50.6.54@o2ib2) [80548.858658] perf: interrupt took too long (3128 > 3127), lowering kernel.perf_event_max_sample_rate to 63000 [81012.175320] Lustre: fir-MDT0003: haven't heard from client 8ec9dd97-d16b-4 (at 10.50.3.47@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbdba4fc00, cur 1594306441 expire 1594306291 last 1594306214 [82938.186116] Lustre: fir-MDT0003: Connection restored to 74a98878-01a2-4 (at 10.50.3.47@o2ib2) [82972.480089] Lustre: fir-MDT0003: Connection restored to 0d87bdec-3769-4 (at 10.50.10.63@o2ib2) [92328.196630] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [96156.601980] Lustre: fir-MDT0003: haven't heard from client e90317e8-753b-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb5c28d000, cur 1594321585 expire 1594321435 last 1594321358 [96156.621870] Lustre: Skipped 1 previous similar message [106922.436688] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [107130.505063] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [112504.045975] Lustre: fir-MDT0003: haven't heard from client cd9aab3e-cddd-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe6a62c00, cur 1594337932 expire 1594337782 last 1594337705 [112554.482787] Lustre: fir-MDT0003: Connection restored to 08be63cd-e3e2-4 (at 10.50.6.54@o2ib2) [135633.644646] Lustre: fir-MDT0003: Client 3d869c09-644e-4 (at 10.50.2.49@o2ib2) reconnecting [135633.653012] Lustre: Skipped 1 previous similar message [135633.658286] Lustre: fir-MDT0003: Connection restored to 3d869c09-644e-4 (at 10.50.2.49@o2ib2) [145200.972840] Lustre: fir-MDT0003: haven't heard from client 2ebed8a9-ecf6-4 (at 10.50.14.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe4a63400, cur 1594370628 expire 1594370478 last 1594370401 [178544.938269] Lustre: fir-MDT0003: haven't heard from client c64f6940-2171-4 (at 10.50.7.17@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf5728000, cur 1594403971 expire 1594403821 last 1594403744 [180229.948644] Lustre: fir-MDT0003: Connection restored to 96628989-a494-4 (at 10.49.26.4@o2ib1) [180820.617907] Lustre: fir-MDT0003: Connection restored to 96628989-a494-4 (at 10.49.26.4@o2ib1) [180871.002775] Lustre: fir-MDT0003: haven't heard from client ec4c390c-6157-4 (at 10.49.26.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef2d0a74400, cur 1594406297 expire 1594406147 last 1594406070 [180871.022777] Lustre: Skipped 3 previous similar messages [180962.273667] Lustre: fir-MDT0003: Connection restored to 1b9e8a72-64fe-4 (at 10.50.3.45@o2ib2) [181015.436345] Lustre: fir-MDT0003: Connection restored to 141698d1-9f0e-4 (at 10.50.16.13@o2ib2) [181312.807437] Lustre: fir-MDT0003: Connection restored to c64f6940-2171-4 (at 10.50.7.17@o2ib2) [181337.016790] Lustre: fir-MDT0003: haven't heard from client 422fcf66-e10f-4 (at 10.49.26.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd9362c00, cur 1594406763 expire 1594406613 last 1594406536 [181709.108404] Lustre: fir-MDT0003: Connection restored to 96628989-a494-4 (at 10.49.26.4@o2ib1) [181709.117022] Lustre: Skipped 1 previous similar message [181780.029093] Lustre: fir-MDT0003: haven't heard from client 5c667095-c13b-4 (at 10.49.26.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe98eb800, cur 1594407206 expire 1594407056 last 1594406979 [182295.814891] Lustre: fir-MDT0003: Connection restored to 96628989-a494-4 (at 10.49.26.4@o2ib1) [182351.045056] Lustre: fir-MDT0003: haven't heard from client 2c0570d0-17b8-4 (at 10.49.26.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebdaf65c00, cur 1594407777 expire 1594407627 last 1594407550 [182677.634872] Lustre: fir-MDT0003: Connection restored to 96628989-a494-4 (at 10.49.26.4@o2ib1) [182737.055594] Lustre: fir-MDT0003: haven't heard from client e5c35181-6d87-4 (at 10.49.26.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfea42c00, cur 1594408163 expire 1594408013 last 1594407936 [183133.526031] Lustre: fir-MDT0003: Connection restored to 96628989-a494-4 (at 10.49.26.4@o2ib1) [183168.068989] Lustre: fir-MDT0003: haven't heard from client 684d099d-0f9c-4 (at 10.49.26.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb7f2aec00, cur 1594408594 expire 1594408444 last 1594408367 [184229.898533] Lustre: fir-MDT0003: Connection restored to 1c060d97-c2ff-4 (at 10.50.9.39@o2ib2) [196957.439651] Lustre: fir-MDT0003: haven't heard from client c0255404-6fe3-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebc275a400, cur 1594422383 expire 1594422233 last 1594422156 [199340.668827] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [199831.518116] Lustre: fir-MDT0003: haven't heard from client 89ca8287-b23c-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf78ae400, cur 1594425257 expire 1594425107 last 1594425030 [199854.030481] Lustre: fir-MDT0003: Connection restored to c0255404-6fe3-4 (at 10.50.8.20@o2ib2) [201745.107886] Lustre: fir-MDT0003: Connection restored to 2ef10909-6964-7c19-1207-151fcbf9efdb (at 10.0.10.4@o2ib7) [204075.065187] Lustre: fir-MDT0003: Connection restored to 2ebed8a9-ecf6-4 (at 10.50.14.3@o2ib2) [208268.254095] Lustre: fir-MDT0003: Connection restored to 14563d49-fbad-4 (at 10.50.10.35@o2ib2) [210675.817340] Lustre: fir-MDT0003: haven't heard from client 125eb3cf-6ffa-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda65f80400, cur 1594436101 expire 1594435951 last 1594435874 [210750.287325] Lustre: fir-MDT0003: Connection restored to 08be63cd-e3e2-4 (at 10.50.6.54@o2ib2) [250267.251371] Lustre: 23955:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594475085/real 1594475085] req@ffff8ecb71f5d580 x1671692405212800/t0(0) o6->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 544/432 e 18 to 1 dl 1594475686 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [250267.279667] Lustre: 23955:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [250267.289508] Lustre: fir-OST0009-osc-MDT0003: Connection to fir-OST0009 (at 10.0.10.102@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [250267.322263] Lustre: fir-OST0009-osc-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [250870.020739] Lustre: 23955:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594475691/real 1594475691] req@ffff8ecb71f5d580 x1671692405212800/t0(0) o6->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 544/432 e 18 to 1 dl 1594476292 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [250870.049045] Lustre: fir-OST0009-osc-MDT0003: Connection to fir-OST0009 (at 10.0.10.102@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [250870.081829] Lustre: fir-OST0009-osc-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [251529.127761] Lustre: 23953:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594476294/real 1594476294] req@ffff8ecb4fe48480 x1671692419711360/t0(0) o6->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 544/432 e 1 to 1 dl 1594476895 ref 1 fl Rpc:X/2/ffffffff rc -11/-1 [251529.156146] Lustre: fir-OST0009-osc-MDT0003: Connection to fir-OST0009 (at 10.0.10.102@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [251529.172775] Lustre: fir-OST0009-osc-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [254828.253328] LNet: Service thread pid 26761 was inactive for 200.31s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [254828.270352] Pid: 26761, comm: mdt03_070 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [254828.280612] Call Trace: [254828.283168] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [254828.290202] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [254828.297412] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [254828.304172] [] osp_md_object_lock+0x162/0x2d0 [osp] [254828.310829] [] lod_object_lock+0xf4/0x780 [lod] [254828.317139] [] mdd_object_lock+0x3e/0xe0 [mdd] [254828.323361] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [254828.330710] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [254828.337538] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [254828.343849] [] mdt_reint_rename+0x2c5/0x2b90 [mdt] [254828.350420] [] mdt_reint_rec+0x83/0x210 [mdt] [254828.356554] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [254828.363210] [] mdt_reint+0x67/0x140 [mdt] [254828.368993] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [254828.376046] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [254828.383849] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [254828.390275] [] kthread+0xd1/0xe0 [254828.395279] [] ret_from_fork_nospec_begin+0xe/0x21 [254828.401872] [] 0xffffffffffffffff [254828.406982] LustreError: dumping log to /tmp/lustre-log.1594480252.26761 [254828.420037] Pid: 26543, comm: mdt03_041 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [254828.430312] Call Trace: [254828.432864] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [254828.439896] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [254828.447091] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [254828.453843] [] osp_md_object_lock+0x162/0x2d0 [osp] [254828.460504] [] lod_object_lock+0xf4/0x780 [lod] [254828.466819] [] mdd_object_lock+0x3e/0xe0 [mdd] [254828.473071] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [254828.480425] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [254828.487254] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [254828.493563] [] mdt_reint_rename+0x2c5/0x2b90 [mdt] [254828.500133] [] mdt_reint_rec+0x83/0x210 [mdt] [254828.506268] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [254828.512926] [] mdt_reint+0x67/0x140 [mdt] [254828.518714] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [254828.525764] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [254828.533597] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [254828.540018] [] kthread+0xd1/0xe0 [254828.545018] [] ret_from_fork_nospec_begin+0xe/0x21 [254828.551595] [] 0xffffffffffffffff [254831.325415] LNet: Service thread pid 55137 was inactive for 200.32s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [254831.342444] LNet: Skipped 1 previous similar message [254831.347505] Pid: 55137, comm: mdt03_106 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [254831.357761] Call Trace: [254831.360312] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [254831.367344] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [254831.374555] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [254831.381318] [] osp_md_object_lock+0x162/0x2d0 [osp] [254831.387993] [] lod_object_lock+0xf4/0x780 [lod] [254831.394298] [] mdd_object_lock+0x3e/0xe0 [mdd] [254831.400527] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [254831.407879] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [254831.414718] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [254831.421029] [] mdt_reint_rename+0x2c5/0x2b90 [mdt] [254831.427613] [] mdt_reint_rec+0x83/0x210 [mdt] [254831.433743] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [254831.440408] [] mdt_reint+0x67/0x140 [mdt] [254831.446202] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [254831.453250] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [254831.461051] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [254831.467466] [] kthread+0xd1/0xe0 [254831.472468] [] ret_from_fork_nospec_begin+0xe/0x21 [254831.479028] [] 0xffffffffffffffff [254831.484150] LustreError: dumping log to /tmp/lustre-log.1594480255.55137 [254836.854704] LNet: Service thread pid 26761 completed after 208.91s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [254836.870947] LNet: Skipped 2 previous similar messages [256309.503072] LustreError: 26581:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'Aquila_results_Monkeys' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 Aquila_results_Monkeys' to finish migration. [256312.794004] LustreError: 26507:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'S24385_lysis_2_hg19' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 S24385_lysis_2_hg19' to finish migration. [260456.829598] LNet: Service thread pid 56147 was inactive for 200.35s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [260456.846621] Pid: 56147, comm: mdt00_055 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [260456.856882] Call Trace: [260456.859438] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [260456.866473] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [260456.873664] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [260456.880427] [] osp_md_object_lock+0x162/0x2d0 [osp] [260456.887082] [] lod_object_lock+0xf4/0x780 [lod] [260456.893410] [] mdd_object_lock+0x3e/0xe0 [mdd] [260456.899632] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [260456.906995] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [260456.913818] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [260456.920144] [] mdt_reint_rename+0x2c5/0x2b90 [mdt] [260456.926716] [] mdt_reint_rec+0x83/0x210 [mdt] [260456.932867] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [260456.939517] [] mdt_reint+0x67/0x140 [mdt] [260456.945297] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [260456.952335] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [260456.960148] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [260456.966575] [] kthread+0xd1/0xe0 [260456.971574] [] ret_from_fork_nospec_begin+0xe/0x21 [260456.978136] [] 0xffffffffffffffff [260456.983258] LustreError: dumping log to /tmp/lustre-log.1594485880.56147 [260456.990679] Pid: 26726, comm: mdt00_024 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [260457.000935] Call Trace: [260457.003487] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [260457.010506] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [260457.017702] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [260457.024451] [] osp_md_object_lock+0x162/0x2d0 [osp] [260457.031101] [] lod_object_lock+0xf4/0x780 [lod] [260457.037400] [] mdd_object_lock+0x3e/0xe0 [mdd] [260457.043615] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [260457.050966] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [260457.057810] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [260457.064122] [] mdt_reint_rename+0x2c5/0x2b90 [mdt] [260457.070696] [] mdt_reint_rec+0x83/0x210 [mdt] [260457.076826] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [260457.083490] [] mdt_reint+0x67/0x140 [mdt] [260457.089272] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [260457.096300] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [260457.104102] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [260457.110515] [] kthread+0xd1/0xe0 [260457.115515] [] ret_from_fork_nospec_begin+0xe/0x21 [260457.122086] [] 0xffffffffffffffff [260473.726080] LNet: Service thread pid 26411 was inactive for 214.45s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [260473.743108] LNet: Skipped 1 previous similar message [260473.748166] Pid: 26411, comm: mdt01_021 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [260473.758444] Call Trace: [260473.761002] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [260473.768036] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [260473.775245] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [260473.782007] [] osp_md_object_lock+0x162/0x2d0 [osp] [260473.788655] [] lod_object_lock+0xf4/0x780 [lod] [260473.794963] [] mdd_object_lock+0x3e/0xe0 [mdd] [260473.801186] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [260473.808536] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [260473.815366] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [260473.821673] [] mdt_reint_rename+0x2c5/0x2b90 [mdt] [260473.828244] [] mdt_reint_rec+0x83/0x210 [mdt] [260473.834381] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [260473.841053] [] mdt_reint+0x67/0x140 [mdt] [260473.846845] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [260473.853889] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [260473.861691] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [260473.868119] [] kthread+0xd1/0xe0 [260473.873123] [] ret_from_fork_nospec_begin+0xe/0x21 [260473.879696] [] 0xffffffffffffffff [260473.884809] LustreError: dumping log to /tmp/lustre-log.1594485897.26411 [260556.476412] Lustre: fir-MDT0000-osp-MDT0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [260556.492510] LustreError: 56147:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1594485680, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ec741f572c0/0xd0d8617a57e7efb8 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x1000001000000 nid: local remote: 0x86a5386a5810ceac expref: -99 pid: 56147 timeout: 0 lvb_type: 0 [260556.562950] Lustre: fir-MDT0000-osp-MDT0003: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [260559.270491] LustreError: 26411:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1594485683, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ee95faf72c0/0xd0d8617a57ee7872 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x1000001000000 nid: local remote: 0x86a5386a581194f1 expref: -99 pid: 26411 timeout: 0 lvb_type: 0 [260559.310573] LustreError: 26411:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 1 previous similar message [260795.275130] LustreError: 26713:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1594485919, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8eca82d4e540/0xd0d8617a59ec2c5b lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 8 type: IBT flags: 0x1000001000000 nid: local remote: 0x86a5386a5821f518 expref: -99 pid: 26713 timeout: 0 lvb_type: 0 [260795.972903] LNet: Service thread pid 56147 completed after 539.48s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [260795.989151] LNet: Skipped 2 previous similar messages [270173.507946] Lustre: fir-MDT0003: haven't heard from client 65131851-4da5-4 (at 10.51.0.67@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf95fc800, cur 1594495597 expire 1594495447 last 1594495370 [283476.868227] Lustre: fir-MDT0003: haven't heard from client 1c9ec9df-8e8b-4 (at 10.49.7.5@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbd6213800, cur 1594508900 expire 1594508750 last 1594508673 [284187.647193] Lustre: fir-MDT0003: Connection restored to 1c9ec9df-8e8b-4 (at 10.49.7.5@o2ib1) [284358.892030] Lustre: fir-MDT0003: haven't heard from client a3c0a565-c6c8-4 (at 10.49.7.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbf180bc00, cur 1594509782 expire 1594509632 last 1594509555 [284503.030689] Lustre: fir-MDT0003: Connection restored to a3c0a565-c6c8-4 (at 10.49.7.4@o2ib1) [294092.019024] LustreError: 26458:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'kousik' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 kousik' to finish migration. [294823.583105] LustreError: 24075:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'madx25' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 madx25' to finish migration. [294892.187196] Lustre: fir-MDT0003: haven't heard from client 61d14e06-24a2-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe1664c00, cur 1594520315 expire 1594520165 last 1594520088 [295141.804718] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [295656.836679] LustreError: 26467:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'metadata' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 metadata' to finish migration. [301101.895701] LustreError: 26491:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'readthrough' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 readthrough' to finish migration. [301195.623038] LustreError: 26385:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'mergedPairs_pear' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 mergedPairs_pear' to finish migration. [301195.640320] LustreError: 26385:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 2 previous similar messages [301337.581684] LustreError: 26425:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'star_gencodeV29' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 star_gencodeV29' to finish migration. [301337.598802] LustreError: 26425:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 1 previous similar message [331735.236178] Lustre: fir-MDT0003: haven't heard from client 89f3381f-cb8e-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1971a5800, cur 1594557157 expire 1594557007 last 1594556930 [332077.054887] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [342385.546614] Lustre: fir-MDT0003: haven't heard from client 8f0671a1-d52d-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecab2d8b000, cur 1594567807 expire 1594567657 last 1594567580 [342452.308211] Lustre: fir-MDT0003: Connection restored to 08be63cd-e3e2-4 (at 10.50.6.54@o2ib2) [342978.578491] Lustre: 24156:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594567799/real 1594567799] req@ffff8ecbdb6ef980 x1671696135338496/t0(0) o5->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 432/432 e 5 to 1 dl 1594568400 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [342978.606794] Lustre: fir-OST0009-osc-MDT0003: Connection to fir-OST0009 (at 10.0.10.102@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [342978.623056] LustreError: 24156:0:(osp_precreate.c:680:osp_precreate_send()) fir-OST0009-osc-MDT0003: can't precreate: rc = -11 [342978.623176] Lustre: fir-OST0009-osc-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [342978.644445] LustreError: 24156:0:(osp_precreate.c:1343:osp_precreate_thread()) fir-OST0009-osc-MDT0003: cannot precreate objects: rc = -11 [343473.537833] Lustre: 23961:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594568400/real 1594568400] req@ffff8eca81b1c800 x1671696135738176/t0(0) o6->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 544/432 e 0 to 1 dl 1594568882 ref 1 fl Rpc:X/2/ffffffff rc -11/-1 [343473.566223] Lustre: fir-OST0009-osc-MDT0003: Connection to fir-OST0009 (at 10.0.10.102@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [343473.582614] Lustre: fir-OST0009-osc-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [343734.678408] Lustre: 24156:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594568400/real 1594568400] req@ffff8ec8c1426c00 x1671696136265152/t0(0) o5->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 432/432 e 0 to 1 dl 1594569156 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [343734.706716] LustreError: 24156:0:(osp_precreate.c:964:osp_precreate_cleanup_orphans()) fir-OST0009-osc-MDT0003: cannot cleanup orphans: rc = -107 [344169.703003] Lustre: 23958:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1594568895/real 1594568895] req@ffff8ec90e0a4380 x1671696135985280/t0(0) o6->fir-OST0009-osc-MDT0003@10.0.10.102@o2ib7:28/4 lens 544/432 e 0 to 1 dl 1594569564 ref 1 fl Rpc:X/2/ffffffff rc -11/-1 [344169.731414] Lustre: fir-OST0009-osc-MDT0003: Connection to fir-OST0009 (at 10.0.10.102@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [344169.747801] Lustre: fir-OST0009-osc-MDT0003: Connection restored to 10.0.10.102@o2ib7 (at 10.0.10.102@o2ib7) [348911.323416] LustreError: 26410:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'readthrough' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 readthrough' to finish migration. [348911.339837] LustreError: 26410:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 2 previous similar messages [357561.800738] Lustre: fir-MDT0003: Client a977620a-38b5-4 (at 10.49.13.28@o2ib1) reconnecting [357561.809221] Lustre: fir-MDT0003: Connection restored to a977620a-38b5-4 (at 10.49.13.28@o2ib1) [357574.726124] Lustre: fir-MDT0003: Client 8caf45d9-f169-4 (at 10.49.20.31@o2ib1) reconnecting [357574.734591] Lustre: fir-MDT0003: Connection restored to 8caf45d9-f169-4 (at 10.49.20.31@o2ib1) [357680.757042] Lustre: fir-MDT0003: Client a977620a-38b5-4 (at 10.49.13.28@o2ib1) reconnecting [357680.765520] Lustre: fir-MDT0003: Connection restored to a977620a-38b5-4 (at 10.49.13.28@o2ib1) [358499.987393] Lustre: fir-MDT0003: haven't heard from client b3b8b740-1934-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa726c00, cur 1594583921 expire 1594583771 last 1594583694 [358850.748538] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [371826.316568] Lustre: fir-MDT0003: Connection restored to f7e1a68a-351b-4 (at 10.50.5.29@o2ib2) [371875.350411] Lustre: fir-MDT0003: haven't heard from client f7e1a68a-351b-4 (at 10.50.5.29@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf9a9b000, cur 1594597296 expire 1594597146 last 1594597069 [372170.053772] Lustre: fir-MDT0003: Connection restored to cb7b8f42-0024-4 (at 10.50.5.56@o2ib2) [372178.843098] Lustre: fir-MDT0003: Connection restored to 7ed6edff-3703-4 (at 10.50.5.40@o2ib2) [372228.361071] Lustre: fir-MDT0003: haven't heard from client cb7b8f42-0024-4 (at 10.50.5.56@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9d40000, cur 1594597649 expire 1594597499 last 1594597422 [399488.556621] Lustre: fir-MDT0003: Client 412b445b-8964-4 (at 10.50.0.63@o2ib2) reconnecting [399488.565004] Lustre: fir-MDT0003: Connection restored to 412b445b-8964-4 (at 10.50.0.63@o2ib2) [399488.573617] Lustre: Skipped 1 previous similar message [399789.149359] Lustre: fir-MDT0003: haven't heard from client 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbeed8fc00, cur 1594625209 expire 1594625059 last 1594624982 [399789.169419] Lustre: Skipped 2 previous similar messages [400192.067472] Lustre: fir-MDT0003: Client 412b445b-8964-4 (at 10.50.0.63@o2ib2) reconnecting [400192.075856] Lustre: fir-MDT0003: Connection restored to 412b445b-8964-4 (at 10.50.0.63@o2ib2) [405623.731403] Lustre: fir-MDT0003: Client 412b445b-8964-4 (at 10.50.0.63@o2ib2) reconnecting [405623.739783] Lustre: fir-MDT0003: Connection restored to 412b445b-8964-4 (at 10.50.0.63@o2ib2) [428363.844394] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [433061.923923] Lustre: fir-MDT0003: Client 0da406f1-5bcc-4 (at 10.50.0.61@o2ib2) reconnecting [433061.932297] Lustre: fir-MDT0003: Connection restored to 0da406f1-5bcc-4 (at 10.50.0.61@o2ib2) [434487.074889] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [435154.152523] Lustre: fir-MDT0003: haven't heard from client ba840f9f-2305-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfa357800, cur 1594660573 expire 1594660423 last 1594660346 [436184.181317] Lustre: fir-MDT0003: haven't heard from client 51373d30-6d3d-4 (at 10.50.9.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf4f79800, cur 1594661603 expire 1594661453 last 1594661376 [437905.903405] Lustre: fir-MDT0003: Connection restored to a357a074-37c4-4 (at 10.50.7.9@o2ib2) [437929.801496] Lustre: fir-MDT0003: Connection restored to ac20933f-e779-4 (at 10.50.9.27@o2ib2) [438078.792079] Lustre: fir-MDT0003: Connection restored to de90a29e-0a50-4 (at 10.50.9.37@o2ib2) [438423.146769] Lustre: fir-MDT0003: Connection restored to 80a33e18-0dab-4 (at 10.50.10.10@o2ib2) [438435.360093] Lustre: fir-MDT0003: Connection restored to 4afecef8-b0db-4 (at 10.50.10.52@o2ib2) [438435.368792] Lustre: Skipped 1 previous similar message [438445.164530] Lustre: fir-MDT0003: Connection restored to c8be1704-fc4a-4 (at 10.50.6.59@o2ib2) [438445.173144] Lustre: Skipped 2 previous similar messages [438482.450595] Lustre: fir-MDT0003: Connection restored to fa17dcf2-b7ef-4 (at 10.50.10.29@o2ib2) [438482.459301] Lustre: Skipped 1 previous similar message [438528.045168] Lustre: fir-MDT0003: Connection restored to 2b622630-5c44-4 (at 10.50.6.4@o2ib2) [438528.053710] Lustre: Skipped 7 previous similar messages [438604.404045] Lustre: fir-MDT0003: Connection restored to 54b5aab9-f1a1-4 (at 10.50.10.34@o2ib2) [438604.412758] Lustre: Skipped 2 previous similar messages [438740.212348] Lustre: fir-MDT0003: Connection restored to 3e769cb6-7af2-4 (at 10.50.9.28@o2ib2) [438740.221002] Lustre: Skipped 70 previous similar messages [438953.257590] Lustre: fir-MDT0003: haven't heard from client c7f01516-803d-4 (at 10.50.7.25@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfea40800, cur 1594664372 expire 1594664222 last 1594664145 [438953.277561] Lustre: Skipped 12 previous similar messages [438999.164793] Lustre: fir-MDT0003: Connection restored to 46d81fc5-42af-4 (at 10.50.9.26@o2ib2) [438999.173430] Lustre: Skipped 85 previous similar messages [439547.131631] Lustre: fir-MDT0003: Connection restored to 5ecb9505-c0c9-4 (at 10.50.2.30@o2ib2) [439547.140248] Lustre: Skipped 20 previous similar messages [440204.942320] Lustre: fir-MDT0003: Connection restored to c3c04515-6bb5-4 (at 10.50.7.58@o2ib2) [440204.950941] Lustre: Skipped 2 previous similar messages [440816.248590] Lustre: fir-MDT0003: Connection restored to 27cfb305-6bd8-4 (at 10.50.2.39@o2ib2) [440816.257205] Lustre: Skipped 5 previous similar messages [442366.698741] Lustre: fir-MDT0003: Connection restored to 4049a2f2-a55c-4 (at 10.49.0.11@o2ib1) [443549.206942] Lustre: fir-MDT0003: Connection restored to a385f7e3-1c05-4 (at 10.50.7.42@o2ib2) [443628.488188] Lustre: fir-MDT0003: Connection restored to 8c0fa524-7dbc-4 (at 10.50.7.41@o2ib2) [444580.996313] Lustre: fir-MDT0003: Connection restored to 32b3c36e-ea5f-4 (at 10.50.7.38@o2ib2) [446596.941945] Lustre: fir-MDT0003: Connection restored to 1752b6bc-2348-4 (at 10.50.7.49@o2ib2) [446760.625260] Lustre: fir-MDT0003: Connection restored to bc610626-b564-4 (at 10.50.6.55@o2ib2) [447114.832354] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [447205.627464] Lustre: fir-MDT0003: Connection restored to 65131851-4da5-4 (at 10.51.0.67@o2ib3) [447472.493664] Lustre: fir-MDT0003: haven't heard from client f52cd336-fde1-4 (at 10.51.0.67@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef7fa7a2800, cur 1594672891 expire 1594672741 last 1594672664 [447472.513638] Lustre: Skipped 1 previous similar message [447756.501068] Lustre: fir-MDT0003: haven't heard from client fdcfb9d6-faac-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe56d8000, cur 1594673175 expire 1594673025 last 1594672948 [447966.221023] Lustre: fir-MDT0003: Connection restored to 706a0027-6df9-4 (at 10.50.10.26@o2ib2) [447966.800906] Lustre: fir-MDT0003: Connection restored to 4394aaac-1d2b-4 (at 10.50.10.32@o2ib2) [447966.809607] Lustre: Skipped 1 previous similar message [447980.975466] Lustre: fir-MDT0003: Connection restored to bf7a98ed-93a7-4 (at 10.50.7.7@o2ib2) [447980.984005] Lustre: Skipped 5 previous similar messages [447983.524278] Lustre: fir-MDT0003: Connection restored to c8be1704-fc4a-4 (at 10.50.6.59@o2ib2) [447983.532896] Lustre: Skipped 3 previous similar messages [447989.948601] Lustre: fir-MDT0003: Connection restored to 2b622630-5c44-4 (at 10.50.6.4@o2ib2) [447989.957137] Lustre: Skipped 1 previous similar message [448009.049047] Lustre: fir-MDT0003: Connection restored to 2e611b3f-a68d-4 (at 10.50.9.57@o2ib2) [448009.057661] Lustre: Skipped 1 previous similar message [448028.841987] Lustre: fir-MDT0003: Connection restored to 6e181de4-ee15-4 (at 10.50.9.19@o2ib2) [448028.850603] Lustre: Skipped 4 previous similar messages [448121.559785] Lustre: fir-MDT0003: Connection restored to 68c77f10-0fea-4 (at 10.50.6.56@o2ib2) [448121.568402] Lustre: Skipped 5 previous similar messages [448375.550774] Lustre: fir-MDT0003: Connection restored to 3824f57b-e755-4 (at 10.50.7.33@o2ib2) [448688.391923] Lustre: fir-MDT0003: Connection restored to 202114aa-d5e2-4 (at 10.51.0.65@o2ib3) [448688.400543] Lustre: Skipped 1 previous similar message [449045.105631] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [449045.114252] Lustre: Skipped 1 previous similar message [449067.536346] Lustre: fir-MDT0003: haven't heard from client f35f490c-5a4a-4 (at 10.50.9.19@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7887000, cur 1594674486 expire 1594674336 last 1594674259 [449536.547973] Lustre: fir-MDT0003: haven't heard from client bbe84672-c7f6-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef4866c1800, cur 1594674955 expire 1594674805 last 1594674728 [449536.567949] Lustre: Skipped 27 previous similar messages [450759.845370] Lustre: fir-MDT0003: Connection restored to 38377cba-5e9f-4 (at 10.50.6.61@o2ib2) [450759.853986] Lustre: Skipped 28 previous similar messages [451286.289956] Lustre: fir-MDT0003: Connection restored to b7f88046-e55d-4 (at 10.50.6.71@o2ib2) [451636.831713] Lustre: fir-MDT0003: Connection restored to 57a495b1-7fa6-4 (at 10.51.0.2@o2ib3) [452182.254056] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [452623.631482] Lustre: fir-MDT0003: haven't heard from client 97283a25-d985-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8f4aad000, cur 1594678042 expire 1594677892 last 1594677815 [452699.031389] Lustre: fir-MDT0003: Connection restored to 1de11674-4375-4 (at 10.50.2.26@o2ib2) [453073.644774] Lustre: fir-MDT0003: haven't heard from client f662a5ce-6942-4 (at 10.50.2.15@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9c4a000, cur 1594678492 expire 1594678342 last 1594678265 [453073.664744] Lustre: Skipped 1 previous similar message [453925.654354] Lustre: fir-MDT0003: Connection restored to fb48c837-7f1a-4 (at 10.50.5.49@o2ib2) [455025.380003] Lustre: fir-MDT0003: Connection restored to 5e00b3ae-fd9c-4 (at 10.50.7.52@o2ib2) [455125.971925] Lustre: fir-MDT0003: Connection restored to 942f8e69-e8bd-4 (at 10.50.10.20@o2ib2) [455395.120140] Lustre: fir-MDT0003: Connection restored to 75830ecc-fc74-4 (at 10.50.7.50@o2ib2) [456056.016637] Lustre: fir-MDT0003: Connection restored to cdfa1e03-f1e9-4 (at 10.50.2.13@o2ib2) [456111.995383] Lustre: fir-MDT0003: Connection restored to 999c2b69-a13d-4 (at 10.50.2.14@o2ib2) [456227.053763] Lustre: fir-MDT0003: Connection restored to 65131851-4da5-4 (at 10.51.0.67@o2ib3) [456363.211228] Lustre: fir-MDT0003: Connection restored to f662a5ce-6942-4 (at 10.50.2.15@o2ib2) [456453.738582] Lustre: fir-MDT0003: haven't heard from client e0baeae7-c072-4 (at 10.51.0.67@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1b2e5cc00, cur 1594681872 expire 1594681722 last 1594681645 [456493.551973] Lustre: fir-MDT0003: Connection restored to ec09a99f-3170-4 (at 10.50.9.12@o2ib2) [456966.884701] Lustre: fir-MDT0003: Connection restored to d1ee7d9e-e069-4 (at 10.50.9.14@o2ib2) [457370.062195] Lustre: fir-MDT0003: Connection restored to 65131851-4da5-4 (at 10.51.0.67@o2ib3) [457618.772164] Lustre: fir-MDT0003: haven't heard from client d4266f6b-4e47-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe35b0c00, cur 1594683037 expire 1594682887 last 1594682810 [458404.805148] Lustre: fir-MDT0003: Connection restored to f662a5ce-6942-4 (at 10.50.2.15@o2ib2) [458404.813768] Lustre: Skipped 1 previous similar message [458630.057244] Lustre: fir-MDT0003: Connection restored to b5cff606-f853-4 (at 10.51.12.19@o2ib3) [458958.625186] Lustre: fir-MDT0003: Connection restored to 8e1f64e3-5416-4 (at 10.49.21.2@o2ib1) [458958.633807] Lustre: Skipped 1 previous similar message [459727.830420] Lustre: fir-MDT0003: haven't heard from client d52b9b9f-55a6-4 (at 10.49.29.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe455ac00, cur 1594685146 expire 1594684996 last 1594684919 [459727.850394] Lustre: Skipped 1 previous similar message [459763.952611] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [459763.961230] Lustre: Skipped 1 previous similar message [459950.846024] Lustre: fir-MDT0003: haven't heard from client 0ead1dc5-d75e-4 (at 10.49.7.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf41dd800, cur 1594685369 expire 1594685219 last 1594685142 [459950.865908] Lustre: Skipped 1 previous similar message [459984.636720] Lustre: fir-MDT0003: Connection restored to 9154a856-2e03-4 (at 10.49.29.7@o2ib1) [460342.720284] Lustre: fir-MDT0003: Connection restored to 8dbf77a1-42bb-4 (at 10.50.6.12@o2ib2) [460968.388395] Lustre: fir-MDT0003: Connection restored to d52b9b9f-55a6-4 (at 10.49.29.8@o2ib1) [460968.397010] Lustre: Skipped 2 previous similar messages [461515.162244] Lustre: fir-MDT0003: Connection restored to b51f4e51-659b-4 (at 10.50.10.43@o2ib2) [461917.277769] Lustre: fir-MDT0003: Connection restored to 486ed06a-9b2d-4 (at 10.50.2.10@o2ib2) [461917.286380] Lustre: Skipped 2 previous similar messages [462610.332832] Lustre: fir-MDT0003: Connection restored to 1adc9579-8e35-4 (at 10.50.2.18@o2ib2) [462610.341449] Lustre: Skipped 1 previous similar message [463711.631161] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [463711.639774] Lustre: Skipped 2 previous similar messages [463970.950909] Lustre: fir-MDT0003: haven't heard from client 4ca5582d-ff67-4 (at 10.50.6.53@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebdab2dc00, cur 1594689389 expire 1594689239 last 1594689162 [466322.526011] Lustre: fir-MDT0003: Connection restored to 4ca5582d-ff67-4 (at 10.50.6.53@o2ib2) [466428.330570] Lustre: fir-MDT0003: Connection restored to eb9b6784-fe22-4 (at 10.50.4.21@o2ib2) [468700.882840] Lustre: fir-MDT0003: Connection restored to 3f736125-a311-4 (at 10.50.2.40@o2ib2) [468700.891457] Lustre: Skipped 1 previous similar message [469180.846632] Lustre: fir-MDT0003: Connection restored to ee089028-3793-4 (at 10.50.7.51@o2ib2) [469355.133774] Lustre: fir-MDT0003: haven't heard from client 671e3b0a-3ed9-4 (at 10.50.15.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ebc24642400, cur 1594694773 expire 1594694623 last 1594694546 [470237.135856] Lustre: fir-MDT0003: haven't heard from client 763df2e8-5c6d-4 (at 10.50.13.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe82fd400, cur 1594695655 expire 1594695505 last 1594695428 [471847.757414] Lustre: fir-MDT0003: Connection restored to 763df2e8-5c6d-4 (at 10.50.13.12@o2ib2) [471986.186549] Lustre: fir-MDT0003: haven't heard from client 83309fc4-0f99-4 (at 10.49.30.22@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba52bc00, cur 1594697404 expire 1594697254 last 1594697177 [472567.204784] Lustre: fir-MDT0003: Connection restored to 671e3b0a-3ed9-4 (at 10.50.15.9@o2ib2) [472584.106564] Lustre: fir-MDT0003: Connection restored to 2d080010-5bb6-4 (at 10.50.6.40@o2ib2) [473224.710868] Lustre: fir-MDT0003: Connection restored to 6e0c9e33-bc6a-4 (at 10.50.10.9@o2ib2) [473587.186580] Lustre: fir-MDT0003: Connection restored to 2a784552-404b-4 (at 10.49.21.17@o2ib1) [473647.021845] Lustre: fir-MDT0003: Connection restored to 22b210ac-ea92-4 (at 10.49.30.12@o2ib1) [473655.166903] Lustre: fir-MDT0003: Connection restored to 32ee4df7-f884-4 (at 10.49.30.24@o2ib1) [473666.890168] Lustre: fir-MDT0003: Connection restored to 627bf0ac-55b3-4 (at 10.49.21.34@o2ib1) [473679.688630] Lustre: fir-MDT0003: Connection restored to f5b0e9ac-db74-4 (at 10.49.21.22@o2ib1) [473679.697337] Lustre: Skipped 2 previous similar messages [473696.587104] Lustre: fir-MDT0003: Connection restored to e7d1779e-bbf2-4 (at 10.49.30.13@o2ib1) [473696.595802] Lustre: Skipped 6 previous similar messages [473730.752326] Lustre: fir-MDT0003: Connection restored to 627e4d63-7d5b-4 (at 10.49.21.15@o2ib1) [473730.761028] Lustre: Skipped 5 previous similar messages [475250.375729] Lustre: fir-MDT0003: Connection restored to 2256468d-1169-4 (at 10.50.6.39@o2ib2) [476028.834914] Lustre: fir-MDT0003: Connection restored to af8ee50c-127d-4 (at 10.49.19.7@o2ib1) [478191.975348] Lustre: fir-MDT0003: Connection restored to 4190de70-07e8-4 (at 10.50.7.45@o2ib2) [478220.850998] Lustre: fir-MDT0003: Connection restored to ae847d21-5787-4 (at 10.50.7.40@o2ib2) [478250.122363] Lustre: fir-MDT0003: Connection restored to 1e6aadc2-2e64-4 (at 10.50.10.6@o2ib2) [478277.523160] Lustre: fir-MDT0003: Connection restored to e5688093-382d-4 (at 10.50.7.47@o2ib2) [478277.531778] Lustre: Skipped 1 previous similar message [478391.270106] Lustre: fir-MDT0003: Connection restored to d1631371-5661-4 (at 10.50.7.46@o2ib2) [479910.157157] Lustre: fir-MDT0003: Connection restored to b4c7f474-4d11-4 (at 10.50.2.12@o2ib2) [482159.134128] Lustre: fir-MDT0003: Connection restored to 7223d5e6-762e-4 (at 10.50.7.6@o2ib2) [483171.741271] Lustre: fir-MDT0003: Connection restored to bee9132c-7d26-4 (at 10.50.9.17@o2ib2) [485582.408744] Lustre: fir-MDT0003: Connection restored to cdf22c7e-ad28-4 (at 10.50.7.39@o2ib2) [490968.943501] Lustre: fir-MDT0003: Connection restored to f36f3074-d851-4 (at 10.50.7.61@o2ib2) [491164.160015] Lustre: fir-MDT0003: Connection restored to 0a3783dd-5599-4 (at 10.50.7.63@o2ib2) [491694.728217] Lustre: fir-MDT0003: Connection restored to 15dec05d-b0cd-4 (at 10.50.2.70@o2ib2) [491712.225299] Lustre: fir-MDT0003: Connection restored to 633d5db7-d63b-4 (at 10.50.9.8@o2ib2) [497217.498691] Lustre: fir-MDT0003: Connection restored to db9411d2-155b-4 (at 10.50.9.5@o2ib2) [497288.034656] Lustre: fir-MDT0003: Connection restored to fc8e386c-65c8-4 (at 10.49.30.31@o2ib1) [497389.922811] Lustre: fir-MDT0003: Connection restored to 0e43da51-6ccc-4 (at 10.49.30.34@o2ib1) [497396.455650] Lustre: fir-MDT0003: Connection restored to b3ff0c7c-b6fb-4 (at 10.49.30.33@o2ib1) [497396.464356] Lustre: Skipped 1 previous similar message [497420.768562] Lustre: fir-MDT0003: Connection restored to baddebee-22aa-4 (at 10.49.13.29@o2ib1) [497428.890067] Lustre: fir-MDT0003: Connection restored to 5de35e91-b2e5-4 (at 10.49.21.24@o2ib1) [497428.898772] Lustre: Skipped 2 previous similar messages [498052.282034] Lustre: fir-MDT0003: Connection restored to 38d470fb-b823-4 (at 10.50.6.7@o2ib2) [498052.290568] Lustre: Skipped 2 previous similar messages [498570.643921] Lustre: fir-MDT0003: Connection restored to 618092b6-b430-4 (at 10.50.9.7@o2ib2) [498956.722897] Lustre: fir-MDT0003: Connection restored to 7eb99425-a9e6-4 (at 10.50.6.11@o2ib2) [499281.696300] Lustre: fir-MDT0003: Connection restored to 0e43da51-6ccc-4 (at 10.49.30.34@o2ib1) [499335.998062] Lustre: fir-MDT0003: haven't heard from client f226efd7-d6c5-4 (at 10.49.30.34@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee104f30400, cur 1594724753 expire 1594724603 last 1594724526 [502063.734973] Lustre: fir-MDT0003: Connection restored to 4d96a30d-08fa-4 (at 10.50.9.6@o2ib2) [504546.024259] Lustre: fir-MDT0003: Connection restored to d036a661-b3dc-4 (at 10.50.6.8@o2ib2) [508087.902049] Lustre: fir-MDT0003: Connection restored to 6604ae3f-91ff-4 (at 10.50.10.48@o2ib2) [509115.893797] Lustre: fir-MDT0003: Connection restored to fd7e0fc3-29a0-4 (at 10.50.7.48@o2ib2) [509120.288426] Lustre: fir-MDT0003: Connection restored to a04d2b02-ef0d-4 (at 10.50.7.44@o2ib2) [509411.136045] Lustre: fir-MDT0003: Connection restored to b9f85368-74f7-4 (at 10.50.7.43@o2ib2) [511596.662772] Lustre: fir-MDT0003: Connection restored to 157de5e6-b9a0-4 (at 10.50.6.44@o2ib2) [516256.280050] Lustre: fir-MDT0003: Connection restored to 863e0b6a-5c8a-4 (at 10.50.2.72@o2ib2) [520577.418116] Lustre: fir-MDT0003: Connection restored to 65131851-4da5-4 (at 10.51.0.67@o2ib3) [520868.601515] Lustre: fir-MDT0003: haven't heard from client 007c3116-1c72-4 (at 10.51.0.67@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee6dfb1bc00, cur 1594746285 expire 1594746135 last 1594746058 [522180.972697] Lustre: fir-MDT0003: Connection restored to 72197a9f-db40-4 (at 10.49.22.4@o2ib1) [523610.337804] Lustre: fir-MDT0003: Connection restored to d4a07642-8d04-4 (at 10.50.6.37@o2ib2) [523761.708610] Lustre: fir-MDT0003: Connection restored to b1b7554d-83c5-4 (at 10.49.19.6@o2ib1) [524562.783267] Lustre: fir-MDT0003: Connection restored to aa5f8800-a3bf-4 (at 10.49.22.5@o2ib1) [526494.764200] Lustre: fir-MDT0003: haven't heard from client d697be2e-af26-4 (at 10.49.17.24@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf981fc00, cur 1594751911 expire 1594751761 last 1594751684 [527669.872417] Lustre: fir-MDT0003: Connection restored to f98d916e-ea7d-4 (at 10.50.6.38@o2ib2) [527685.725900] Lustre: fir-MDT0003: Client 3d869c09-644e-4 (at 10.50.2.49@o2ib2) reconnecting [527685.734288] Lustre: fir-MDT0003: Connection restored to 3d869c09-644e-4 (at 10.50.2.49@o2ib2) [527722.666677] Lustre: fir-MDT0003: Connection restored to aa2b27fb-be35-4 (at 10.50.13.11@o2ib2) [527743.554669] Lustre: fir-MDT0003: Connection restored to 0e3aeb7a-ae61-4 (at 10.50.15.3@o2ib2) [527765.310395] Lustre: fir-MDT0003: Connection restored to be2724ca-fab1-4 (at 10.50.13.10@o2ib2) [527857.071973] Lustre: fir-MDT0003: Connection restored to 51d5db08-ff65-4 (at 10.50.14.8@o2ib2) [527888.530919] Lustre: fir-MDT0003: Connection restored to e5a9e2af-1e14-4 (at 10.50.13.4@o2ib2) [527921.152257] Lustre: fir-MDT0003: Connection restored to 7e0490c0-bc49-4 (at 10.50.14.13@o2ib2) [527921.160972] Lustre: Skipped 1 previous similar message [527985.855359] Lustre: fir-MDT0003: Connection restored to 484dc59e-8ea0-4 (at 10.50.14.12@o2ib2) [527985.864064] Lustre: Skipped 2 previous similar messages [528184.683877] Lustre: fir-MDT0003: Connection restored to 20098632-6ba5-4 (at 10.50.14.11@o2ib2) [528184.692590] Lustre: Skipped 10 previous similar messages [528454.775934] Lustre: fir-MDT0003: Connection restored to 8144d85e-be2a-4 (at 10.49.28.12@o2ib1) [528454.784641] Lustre: Skipped 19 previous similar messages [530063.519302] Lustre: fir-MDT0003: Connection restored to ea14b1f3-bff6-4 (at 10.50.10.15@o2ib2) [530063.528009] Lustre: Skipped 1 previous similar message [530175.247425] Lustre: fir-MDT0003: Connection restored to e3daed6c-34ed-4 (at 10.50.15.12@o2ib2) [531759.997918] Lustre: fir-MDT0003: Connection restored to da49ce0b-cfe4-4 (at 10.50.6.17@o2ib2) [532353.945384] Lustre: fir-MDT0003: Connection restored to 3d5a41cd-8545-4 (at 10.50.6.18@o2ib2) [532494.726248] Lustre: fir-MDT0003: Connection restored to 0cb6cdd6-fdbf-4 (at 10.50.6.26@o2ib2) [532900.241150] Lustre: fir-MDT0003: Connection restored to 39039706-70df-4 (at 10.49.17.14@o2ib1) [534818.684670] Lustre: fir-MDT0003: Connection restored to 1ed63a93-df65-4 (at 10.49.21.20@o2ib1) [534852.996404] Lustre: fir-MDT0003: haven't heard from client 00abff9b-fdda-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb2629e800, cur 1594760269 expire 1594760119 last 1594760042 [534853.016376] Lustre: Skipped 23 previous similar messages [534960.776216] Lustre: fir-MDT0003: Connection restored to 347f2b86-0638-4 (at 10.49.30.2@o2ib1) [534995.293492] Lustre: fir-MDT0003: Connection restored to 5aa481e2-07c1-4 (at 10.49.30.20@o2ib1) [534995.302191] Lustre: Skipped 6 previous similar messages [536537.194118] Lustre: fir-MDT0003: Connection restored to 68028cfc-4ece-4 (at 10.50.15.1@o2ib2) [536537.202751] Lustre: Skipped 8 previous similar messages [536827.914741] Lustre: fir-MDT0003: Connection restored to 0d205980-565e-4 (at 10.49.30.18@o2ib1) [537555.572076] Lustre: fir-MDT0003: Connection restored to 249e1bca-9877-4 (at 10.49.19.1@o2ib1) [538935.668627] Lustre: fir-MDT0003: Connection restored to 91eb6569-9705-4 (at 10.50.6.16@o2ib2) [539171.576439] Lustre: fir-MDT0003: Connection restored to b6eb2559-ae6e-4 (at 10.50.6.23@o2ib2) [539915.134455] Lustre: fir-MDT0003: haven't heard from client f6f43919-59a9-4 (at 10.50.4.24@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbda50ac00, cur 1594765331 expire 1594765181 last 1594765104 [542013.240066] Lustre: fir-MDT0003: Connection restored to 4e78023b-3181-4 (at 10.50.14.1@o2ib2) [543190.544508] Lustre: fir-MDT0003: Connection restored to 3f8e717b-6e9c-4 (at 10.49.22.3@o2ib1) [544137.253778] Lustre: fir-MDT0003: haven't heard from client 4c6ced91-5175-4 (at 10.49.28.4@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebc275cc00, cur 1594769553 expire 1594769403 last 1594769326 [544694.252439] Lustre: fir-MDT0003: Connection restored to 8ab56a74-861d-4 (at 10.50.9.13@o2ib2) [544834.687202] Lustre: fir-MDT0003: Connection restored to 5fb3cee5-9798-4 (at 10.49.22.18@o2ib1) [545037.291665] Lustre: fir-MDT0003: Connection restored to dbc91735-d2a6-4 (at 10.49.22.6@o2ib1) [545584.821934] Lustre: fir-MDT0003: Connection restored to 1e0f311f-3670-4 (at 10.49.22.7@o2ib1) [545879.471846] Lustre: fir-MDT0003: Connection restored to 4c6ced91-5175-4 (at 10.49.28.4@o2ib1) [546802.075384] Lustre: fir-MDT0003: Connection restored to aabec3aa-b7c0-4 (at 10.50.6.24@o2ib2) [546896.211504] Lustre: fir-MDT0003: Connection restored to 12c40ed0-71af-4 (at 10.49.22.12@o2ib1) [547384.686892] Lustre: fir-MDT0003: Connection restored to 8066e970-61e4-4 (at 10.49.22.10@o2ib1) [547493.349878] Lustre: fir-MDT0003: haven't heard from client 21615c7b-8e08-4 (at 10.50.9.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebece62800, cur 1594772909 expire 1594772759 last 1594772682 [547875.362718] Lustre: fir-MDT0003: haven't heard from client dd8a6296-1aa9-4 (at 10.49.28.1@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf4f7f800, cur 1594773291 expire 1594773141 last 1594773064 [547875.382688] Lustre: Skipped 9 previous similar messages [548387.367454] Lustre: fir-MDT0003: haven't heard from client 7223c0eb-8bc4-4 (at 10.50.14.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbea5fd000, cur 1594773803 expire 1594773653 last 1594773576 [548577.372319] Lustre: fir-MDT0003: haven't heard from client 1ff8f707-228f-4 (at 10.50.14.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf41de000, cur 1594773993 expire 1594773843 last 1594773766 [548809.007103] Lustre: fir-MDT0003: Connection restored to 88c7415b-ad94-4 (at 10.50.6.20@o2ib2) [548811.512677] Lustre: fir-MDT0003: Connection restored to 101764a8-b976-4 (at 10.49.29.1@o2ib1) [549003.470795] Lustre: fir-MDT0003: Connection restored to e9c5ee2d-a6ca-4 (at 10.49.22.1@o2ib1) [549022.617819] Lustre: fir-MDT0003: Connection restored to 8ab56a74-861d-4 (at 10.50.9.13@o2ib2) [549051.006553] Lustre: fir-MDT0003: Connection restored to 6f269912-56af-4 (at 10.49.22.8@o2ib1) [549065.032099] Lustre: fir-MDT0003: Connection restored to 6eec260e-417e-4 (at 10.50.13.9@o2ib2) [549159.075118] Lustre: fir-MDT0003: Connection restored to 0ead1dc5-d75e-4 (at 10.49.7.8@o2ib1) [549207.840822] Lustre: fir-MDT0003: Connection restored to 38377cba-5e9f-4 (at 10.50.6.61@o2ib2) [549207.849442] Lustre: Skipped 3 previous similar messages [549272.885663] Lustre: fir-MDT0003: Connection restored to dd8a6296-1aa9-4 (at 10.49.28.1@o2ib1) [549272.894274] Lustre: Skipped 2 previous similar messages [549406.113057] Lustre: fir-MDT0003: Connection restored to e3daed6c-34ed-4 (at 10.50.15.12@o2ib2) [549406.121765] Lustre: Skipped 3 previous similar messages [549898.165530] Lustre: fir-MDT0003: Connection restored to f1a83b07-70da-4 (at 10.50.10.40@o2ib2) [549898.174230] Lustre: Skipped 2 previous similar messages [550486.287530] Lustre: fir-MDT0003: Connection restored to f247fbe6-1389-4 (at 10.50.6.19@o2ib2) [550486.296143] Lustre: Skipped 2 previous similar messages [551873.466905] Lustre: fir-MDT0003: haven't heard from client 6d5d9b58-c7d2-4 (at 10.50.1.57@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf8bf0000, cur 1594777289 expire 1594777139 last 1594777062 [552180.093668] Lustre: fir-MDT0003: Connection restored to 4d02f8f2-51d2-4 (at 10.50.6.35@o2ib2) [552975.494461] Lustre: fir-MDT0003: haven't heard from client f595c8d8-5ef2-4 (at 10.50.8.57@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe732bc00, cur 1594778391 expire 1594778241 last 1594778164 [552975.514453] Lustre: Skipped 16 previous similar messages [553408.514495] Lustre: fir-MDT0003: haven't heard from client 07c12eb0-a74d-4 (at 10.50.13.2@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbef774400, cur 1594778824 expire 1594778674 last 1594778597 [553408.534466] Lustre: Skipped 1 previous similar message [553484.511799] Lustre: fir-MDT0003: haven't heard from client 0e871928-94aa-4 (at 10.50.8.64@o2ib2) in 155 seconds. I think it's dead, and I am evicting it. exp ffff8eebba573800, cur 1594778900 expire 1594778750 last 1594778745 [553484.531764] Lustre: Skipped 1 previous similar message [553590.139615] Lustre: fir-MDT0003: Connection restored to d3e35bcf-3553-4 (at 10.50.17.4@o2ib2) [553675.125298] Lustre: fir-MDT0003: Connection restored to 9436efeb-8bbe-4 (at 10.50.5.23@o2ib2) [553771.055870] Lustre: fir-MDT0003: Connection restored to 78e8ebc3-fd73-4 (at 10.50.1.60@o2ib2) [553906.759333] Lustre: fir-MDT0003: Connection restored to cf29efca-8d3c-4 (at 10.50.3.46@o2ib2) [554128.582087] Lustre: fir-MDT0003: Connection restored to a7c29338-fa1e-4 (at 10.50.17.13@o2ib2) [554279.959967] Lustre: fir-MDT0003: Connection restored to a7ec475e-2548-4 (at 10.50.17.18@o2ib2) [554279.968676] Lustre: Skipped 4 previous similar messages [554372.534546] Lustre: fir-MDT0003: haven't heard from client f5426f12-a085-4 (at 10.50.8.61@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbda4e7800, cur 1594779788 expire 1594779638 last 1594779561 [554593.506667] Lustre: fir-MDT0003: Connection restored to 7afac300-e91d-4 (at 10.50.3.37@o2ib2) [554593.515287] Lustre: Skipped 53 previous similar messages [554854.546973] Lustre: fir-MDT0003: haven't heard from client c1633fab-f225-4 (at 10.50.4.22@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf46fc800, cur 1594780270 expire 1594780120 last 1594780043 [555153.555470] Lustre: fir-MDT0003: haven't heard from client ac77f0ac-d85d-4 (at 10.50.8.66@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf8844400, cur 1594780569 expire 1594780419 last 1594780342 [555201.057959] Lustre: fir-MDT0003: Connection restored to eb9b6784-fe22-4 (at 10.50.4.21@o2ib2) [555201.066577] Lustre: Skipped 60 previous similar messages [555651.571767] Lustre: fir-MDT0003: haven't heard from client 35ac1379-93ed-4 (at 10.50.8.59@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebdab2f400, cur 1594781067 expire 1594780917 last 1594780840 [555838.575804] Lustre: fir-MDT0003: haven't heard from client 7aa46641-abf6-4 (at 10.50.1.58@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf9ba2800, cur 1594781254 expire 1594781104 last 1594781027 [556019.709340] Lustre: fir-MDT0003: Connection restored to a703e38b-99a7-4 (at 10.50.13.3@o2ib2) [556019.717960] Lustre: Skipped 18 previous similar messages [556972.587938] Lustre: fir-MDT0003: Connection restored to 4441ab51-5055-4 (at 10.50.5.14@o2ib2) [556972.596549] Lustre: Skipped 5 previous similar messages [557647.809961] Lustre: fir-MDT0003: Connection restored to f99a529a-3ad2-4 (at 10.50.4.15@o2ib2) [557647.818611] Lustre: Skipped 7 previous similar messages [558303.301733] Lustre: fir-MDT0003: Connection restored to 35ac1379-93ed-4 (at 10.50.8.59@o2ib2) [558303.310349] Lustre: Skipped 11 previous similar messages [558739.659726] Lustre: fir-MDT0003: haven't heard from client 93dd83f3-813b-4 (at 10.50.15.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfebdd400, cur 1594784155 expire 1594784005 last 1594783928 [559212.673873] Lustre: fir-MDT0003: haven't heard from client e3cbec7b-e25a-4 (at 10.50.8.62@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbd6217000, cur 1594784628 expire 1594784478 last 1594784401 [559366.319669] Lustre: fir-MDT0003: Connection restored to 1e20a039-42e4-4 (at 10.50.1.55@o2ib2) [559366.328283] Lustre: Skipped 3 previous similar messages [560083.339822] Lustre: fir-MDT0003: Connection restored to 93dd83f3-813b-4 (at 10.50.15.5@o2ib2) [560083.348439] Lustre: Skipped 1 previous similar message [560246.705362] Lustre: fir-MDT0003: haven't heard from client 46fcd755-682b-4 (at 10.50.13.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe82f9400, cur 1594785662 expire 1594785512 last 1594785435 [560246.725414] Lustre: Skipped 1 previous similar message [561045.388053] Lustre: fir-MDT0003: Connection restored to e3cbec7b-e25a-4 (at 10.50.8.62@o2ib2) [561655.566678] Lustre: fir-MDT0003: Connection restored to 69dff316-8c11-4 (at 10.50.8.67@o2ib2) [562778.269112] Lustre: fir-MDT0003: Connection restored to 8ae40cd4-4184-4 (at 10.50.17.47@o2ib2) [562778.277810] Lustre: Skipped 1 previous similar message [564533.833027] Lustre: fir-MDT0003: Connection restored to fe9c9aaf-3e75-4 (at 10.50.3.67@o2ib2) [564635.802513] Lustre: fir-MDT0003: Connection restored to b2fea301-c405-4 (at 10.50.13.6@o2ib2) [566818.644237] Lustre: fir-MDT0003: Connection restored to e3ae490a-4c1d-4 (at 10.50.5.11@o2ib2) [566818.652852] Lustre: Skipped 2 previous similar messages [566844.728978] Lustre: fir-MDT0003: Connection restored to f1b08221-f6fa-4 (at 10.50.5.7@o2ib2) [566978.908914] Lustre: fir-MDT0003: haven't heard from client a19fb8dc-9cc0-4 (at 10.50.8.60@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe732dc00, cur 1594792394 expire 1594792244 last 1594792167 [567677.073315] Lustre: fir-MDT0003: Connection restored to 78c24312-cfc6-4 (at 10.49.13.26@o2ib1) [568092.812284] Lustre: fir-MDT0003: Connection restored to 06d630f1-1b85-4 (at 10.50.4.54@o2ib2) [568716.013686] Lustre: fir-MDT0003: Connection restored to 5de35e91-b2e5-4 (at 10.49.21.24@o2ib1) [568973.453624] Lustre: fir-MDT0003: Connection restored to ca57a6a6-f3d4-4 (at 10.50.3.21@o2ib2) [568977.321176] Lustre: fir-MDT0003: Connection restored to 1bfad575-ea6a-4 (at 10.50.3.7@o2ib2) [568983.157641] Lustre: fir-MDT0003: Connection restored to 72b44d7b-b19e-4 (at 10.50.3.8@o2ib2) [568983.166176] Lustre: Skipped 1 previous similar message [569005.490545] Lustre: fir-MDT0003: Connection restored to f21da883-e236-4 (at 10.50.3.4@o2ib2) [569005.499072] Lustre: Skipped 1 previous similar message [569024.480475] Lustre: fir-MDT0003: Connection restored to a9bbab2f-7dde-4 (at 10.50.3.16@o2ib2) [569024.489095] Lustre: Skipped 2 previous similar messages [569126.155643] Lustre: fir-MDT0003: Connection restored to b5d37ab7-1e06-4 (at 10.50.3.13@o2ib2) [569255.785587] Lustre: fir-MDT0003: Connection restored to 83e0c472-90a1-4 (at 10.50.3.10@o2ib2) [569419.380878] Lustre: fir-MDT0003: Connection restored to c9a6e62d-e1e4-4 (at 10.50.17.44@o2ib2) [569419.389583] Lustre: Skipped 4 previous similar messages [570565.876707] Lustre: fir-MDT0003: Connection restored to 48072254-206e-4 (at 10.50.1.62@o2ib2) [570565.885327] Lustre: Skipped 2 previous similar messages [571722.367573] Lustre: fir-MDT0003: Connection restored to d8420d8c-503d-4 (at 10.49.22.24@o2ib1) [571753.374659] Lustre: fir-MDT0003: Connection restored to 11c6c189-a510-4 (at 10.49.22.23@o2ib1) [571770.406312] Lustre: fir-MDT0003: Connection restored to 901ed1f6-2bdd-4 (at 10.49.22.22@o2ib1) [572304.269273] Lustre: fir-MDT0003: Connection restored to 50a2c4ee-26cf-4 (at 10.50.10.42@o2ib2) [573760.646797] Lustre: fir-MDT0003: Connection restored to 46911d4c-8ac0-4 (at 10.50.4.62@o2ib2) [573961.312477] Lustre: fir-MDT0003: Connection restored to 8aafd52d-3a9d-4 (at 10.50.8.40@o2ib2) [573961.321086] Lustre: Skipped 1 previous similar message [574289.448315] Lustre: fir-MDT0003: Connection restored to 6329cf05-27f1-4 (at 10.50.5.13@o2ib2) [576593.150329] Lustre: fir-MDT0003: Connection restored to 5111cae0-a410-4 (at 10.50.2.62@o2ib2) [576823.256276] Lustre: fir-MDT0003: Connection restored to c3cccd5b-a399-4 (at 10.50.2.66@o2ib2) [577783.983162] Lustre: fir-MDT0003: Connection restored to a6a5f65f-00a9-4 (at 10.50.2.68@o2ib2) [577787.225718] Lustre: fir-MDT0003: Connection restored to a8d18520-df88-4 (at 10.50.2.56@o2ib2) [577794.613287] Lustre: fir-MDT0003: Connection restored to 1ca1ab45-a5dc-4 (at 10.50.2.58@o2ib2) [577800.030256] Lustre: fir-MDT0003: Connection restored to 2e8ec445-fa71-4 (at 10.50.2.59@o2ib2) [577800.038872] Lustre: Skipped 1 previous similar message [577812.385559] Lustre: fir-MDT0003: Connection restored to e756f374-c7c9-4 (at 10.50.2.57@o2ib2) [577812.394177] Lustre: Skipped 1 previous similar message [577826.539940] Lustre: fir-MDT0003: Connection restored to 4ee0ee2f-6a23-4 (at 10.50.2.55@o2ib2) [577858.817605] Lustre: fir-MDT0003: Connection restored to 400873ee-7ac3-4 (at 10.50.2.65@o2ib2) [577858.826219] Lustre: Skipped 1 previous similar message [577968.538023] Lustre: fir-MDT0003: Connection restored to 7443cad3-3bc9-4 (at 10.50.5.15@o2ib2) [577968.546643] Lustre: Skipped 2 previous similar messages [578098.079881] Lustre: fir-MDT0003: Connection restored to 74478a25-cda1-4 (at 10.50.2.51@o2ib2) [578399.243533] Lustre: fir-MDT0003: haven't heard from client f06a0743-5add-4 (at 10.50.15.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa690c00, cur 1594803814 expire 1594803664 last 1594803587 [579125.069565] Lustre: fir-MDT0003: Connection restored to c8bc6b6e-7b93-4 (at 10.50.1.38@o2ib2) [579125.078183] Lustre: Skipped 5 previous similar messages [579277.687431] Lustre: fir-MDT0003: Connection restored to 32067c0e-a9e2-4 (at 10.50.1.35@o2ib2) [579552.664167] Lustre: fir-MDT0003: Connection restored to c89a8f30-cd25-4 (at 10.50.4.55@o2ib2) [579711.658174] Lustre: fir-MDT0003: Connection restored to accf69b9-5f9b-4 (at 10.50.15.8@o2ib2) [579711.666794] Lustre: Skipped 1 previous similar message [580541.732648] Lustre: fir-MDT0003: Connection restored to 7f02685c-80f3-4 (at 10.50.1.63@o2ib2) [581120.033530] Lustre: fir-MDT0003: Connection restored to b9b25422-b6e1-4 (at 10.50.6.36@o2ib2) [581507.395834] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [581668.829153] Lustre: fir-MDT0003: Connection restored to 108abc66-db13-4 (at 10.50.2.33@o2ib2) [581668.837781] Lustre: Skipped 1 previous similar message [582589.745175] Lustre: fir-MDT0003: Client 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) reconnecting [582589.753552] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [582616.278896] Lustre: fir-MDT0003: Connection restored to dff47ee2-92a4-4 (at 10.49.31.7@o2ib1) [582754.233181] Lustre: fir-MDT0003: Connection restored to 58687430-74ab-4 (at 10.49.31.2@o2ib1) [583748.206736] Lustre: fir-MDT0003: Client 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) reconnecting [583748.215116] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [583748.223744] Lustre: Skipped 7 previous similar messages [584336.691227] Lustre: fir-MDT0003: Connection restored to a13390c6-5c93-4 (at 10.50.8.36@o2ib2) [584696.766073] Lustre: fir-MDT0003: Connection restored to b2952cb1-5c86-4 (at 10.49.31.1@o2ib1) [584748.080485] Lustre: fir-MDT0003: Connection restored to 58687430-74ab-4 (at 10.49.31.2@o2ib1) [584829.049459] Lustre: fir-MDT0003: Client 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) reconnecting [584829.057833] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [584967.909850] Lustre: fir-MDT0003: Client 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) reconnecting [586843.491915] Lustre: fir-MDT0003: haven't heard from client 8b3a5c8b-1f9b-4 (at 10.50.1.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebdb0ed800, cur 1594812258 expire 1594812108 last 1594812031 [586843.511884] Lustre: Skipped 1 previous similar message [587010.492073] Lustre: fir-MDT0003: haven't heard from client 9a61a03f-cf00-4 (at 10.50.5.34@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf8634c00, cur 1594812425 expire 1594812275 last 1594812198 [587010.512069] Lustre: Skipped 3 previous similar messages [589568.238439] Lustre: fir-MDT0003: Connection restored to 6083ede7-8e62-4 (at 10.50.8.39@o2ib2) [589568.247053] Lustre: Skipped 1 previous similar message [589610.073918] Lustre: fir-MDT0003: Connection restored to 78b6d149-1419-4 (at 10.50.5.16@o2ib2) [589610.082535] Lustre: Skipped 1 previous similar message [589731.374286] Lustre: fir-MDT0003: Connection restored to 306bc6e2-94a6-4 (at 10.50.1.2@o2ib2) [589830.094644] Lustre: fir-MDT0003: Connection restored to a5efd8d2-bcc1-4 (at 10.50.1.10@o2ib2) [589830.103268] Lustre: Skipped 6 previous similar messages [590044.128534] Lustre: fir-MDT0003: Connection restored to 4c0d664f-d491-4 (at 10.50.5.46@o2ib2) [590541.044377] Lustre: fir-MDT0003: Connection restored to 20e04d32-48e5-4 (at 10.50.1.15@o2ib2) [590541.052990] Lustre: Skipped 3 previous similar messages [591597.915526] Lustre: fir-MDT0003: Connection restored to d9f1d79c-ae6f-4 (at 10.50.3.38@o2ib2) [591597.924143] Lustre: Skipped 1 previous similar message [593147.667301] Lustre: fir-MDT0003: haven't heard from client c1202ce4-6a99-4 (at 10.50.5.50@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf8bfc800, cur 1594818562 expire 1594818412 last 1594818335 [593665.840040] Lustre: fir-MDT0003: Connection restored to 9b4040fa-8db7-4 (at 10.50.1.22@o2ib2) [594026.694301] Lustre: fir-MDT0003: haven't heard from client 133973df-7750-4 (at 10.50.5.40@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb43b53800, cur 1594819441 expire 1594819291 last 1594819214 [594169.140388] Lustre: fir-MDT0003: Connection restored to 65cacd4f-016b-4 (at 10.50.1.18@o2ib2) [594369.268320] Lustre: fir-MDT0003: Connection restored to e3c9c259-9ea7-4 (at 10.50.1.64@o2ib2) [595672.883277] Lustre: fir-MDT0003: Connection restored to 12a68337-dfb7-4 (at 10.50.3.25@o2ib2) [596830.798927] Lustre: fir-MDT0003: Connection restored to c1202ce4-6a99-4 (at 10.50.5.50@o2ib2) [597582.801791] Lustre: fir-MDT0003: haven't heard from client 82648ea4-154d-4 (at 10.50.5.56@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90b551c00, cur 1594822997 expire 1594822847 last 1594822770 [598162.985250] Lustre: fir-MDT0003: Connection restored to 439e7670-7246-4 (at 10.50.1.23@o2ib2) [598698.024724] Lustre: fir-MDT0003: Connection restored to 16e5ecfc-d3ba-4 (at 10.50.8.47@o2ib2) [598756.158797] Lustre: fir-MDT0003: Connection restored to ca4139cd-70b7-4 (at 10.50.10.41@o2ib2) [598898.698801] Lustre: fir-MDT0003: Connection restored to f0b3d5ba-9585-4 (at 10.50.8.51@o2ib2) [599109.839377] Lustre: fir-MDT0003: haven't heard from client fd906f0a-9cbf-4 (at 10.50.1.43@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe4a61c00, cur 1594824524 expire 1594824374 last 1594824297 [599117.501797] Lustre: fir-MDT0003: Connection restored to c566add9-5591-4 (at 10.50.8.53@o2ib2) [599163.554215] Lustre: fir-MDT0003: Connection restored to 10302dcd-0971-4 (at 10.50.1.48@o2ib2) [599475.056870] Lustre: fir-MDT0003: Connection restored to 0548873c-547f-4 (at 10.50.1.16@o2ib2) [599475.065486] Lustre: Skipped 1 previous similar message [599816.859285] Lustre: fir-MDT0003: haven't heard from client 92eddfe1-d69b-4 (at 10.50.8.50@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7897400, cur 1594825231 expire 1594825081 last 1594825004 [600777.203255] Lustre: fir-MDT0003: Connection restored to 946e083d-ed37-4 (at 10.50.10.44@o2ib2) [600788.103684] Lustre: fir-MDT0003: Connection restored to 258f5cfb-34ef-4 (at 10.50.10.46@o2ib2) [600848.498732] Lustre: fir-MDT0003: Connection restored to 6f63a640-5846-4 (at 10.50.10.47@o2ib2) [601020.872187] Lustre: fir-MDT0003: Connection restored to 723d24a6-a39b-4 (at 10.50.8.55@o2ib2) [601254.479908] Lustre: fir-MDT0003: Connection restored to fdf9eb72-3824-4 (at 10.50.1.49@o2ib2) [601254.488524] Lustre: Skipped 3 previous similar messages [601548.682159] Lustre: fir-MDT0003: Connection restored to 35a8e232-ad02-4 (at 10.50.5.60@o2ib2) [601741.735642] Lustre: fir-MDT0003: Connection restored to 959e7794-7b00-4 (at 10.50.5.59@o2ib2) [601741.744259] Lustre: Skipped 2 previous similar messages [602065.184350] Lustre: fir-MDT0003: Connection restored to 447ff534-9e82-4 (at 10.50.5.67@o2ib2) [602065.192967] Lustre: Skipped 4 previous similar messages [602926.962948] Lustre: fir-MDT0003: haven't heard from client 92fe5151-7cb4-4 (at 10.50.5.52@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9d45c00, cur 1594828341 expire 1594828191 last 1594828114 [603018.675785] Lustre: fir-MDT0003: Connection restored to 04bcc146-f6f3-4 (at 10.50.3.26@o2ib2) [603018.684405] Lustre: Skipped 2 previous similar messages [604043.982007] Lustre: fir-MDT0003: haven't heard from client 4a1b4e82-b6e8-4 (at 10.50.5.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa7dec00, cur 1594829458 expire 1594829308 last 1594829231 [604119.984883] Lustre: fir-MDT0003: haven't heard from client 2d4c4134-67a0-4 (at 10.50.1.30@o2ib2) in 212 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe4a67800, cur 1594829534 expire 1594829384 last 1594829322 [604259.765776] Lustre: fir-MDT0003: Connection restored to d1752b7f-4b18-4 (at 10.50.7.21@o2ib2) [606122.246399] Lustre: fir-MDT0003: Connection restored to 92fe5151-7cb4-4 (at 10.50.5.52@o2ib2) [607086.265553] Lustre: fir-MDT0003: Connection restored to 2d4c4134-67a0-4 (at 10.50.1.30@o2ib2) [607103.069595] Lustre: fir-MDT0003: haven't heard from client 9019b5d5-ecec-4 (at 10.50.1.36@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9cd2800, cur 1594832517 expire 1594832367 last 1594832290 [607186.389222] Lustre: fir-MDT0003: Connection restored to 76480b18-ef7c-4 (at 10.50.1.13@o2ib2) [607343.075616] Lustre: fir-MDT0003: haven't heard from client 37c881b8-4bcd-4 (at 10.50.1.41@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbea5f8800, cur 1594832757 expire 1594832607 last 1594832530 [607394.786290] Lustre: fir-MDT0003: Connection restored to 8d56db38-4232-4 (at 10.50.1.46@o2ib2) [607548.440868] Lustre: fir-MDT0003: Connection restored to a7c29338-fa1e-4 (at 10.50.17.13@o2ib2) [607655.491600] Lustre: fir-MDT0003: Connection restored to 4a1b4e82-b6e8-4 (at 10.50.5.54@o2ib2) [607655.500219] Lustre: Skipped 2 previous similar messages [608218.108685] Lustre: fir-MDT0003: haven't heard from client e859ab6a-f865-4 (at 10.50.1.31@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe4e93800, cur 1594833632 expire 1594833482 last 1594833405 [608932.126167] Lustre: fir-MDT0003: haven't heard from client db3002b0-b461-4 (at 10.50.1.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe9b3a400, cur 1594834346 expire 1594834196 last 1594834119 [608939.692613] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [609232.140108] Lustre: fir-MDT0003: haven't heard from client 2837e42a-cfda-4 (at 10.50.5.55@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba528c00, cur 1594834646 expire 1594834496 last 1594834419 [610007.703257] Lustre: fir-MDT0003: Connection restored to 9019b5d5-ecec-4 (at 10.50.1.36@o2ib2) [610081.153835] Lustre: fir-MDT0003: haven't heard from client 80fd9564-33c3-4 (at 10.50.5.61@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfa293800, cur 1594835495 expire 1594835345 last 1594835268 [610282.290313] Lustre: fir-MDT0003: Connection restored to 37c881b8-4bcd-4 (at 10.50.1.41@o2ib2) [610351.048389] Lustre: fir-MDT0003: Connection restored to 588a2fd9-7db9-4 (at 10.50.8.48@o2ib2) [610432.093854] Lustre: fir-MDT0003: Connection restored to e30ae087-51c3-4 (at 10.50.17.46@o2ib2) [610534.895733] Lustre: fir-MDT0003: Connection restored to 70e3ce09-5cf3-4 (at 10.50.1.11@o2ib2) [610566.961798] Lustre: fir-MDT0003: Connection restored to 962cad5e-93ee-4 (at 10.50.1.42@o2ib2) [610769.174102] Lustre: fir-MDT0003: haven't heard from client 209ce0d6-5cde-4 (at 10.50.17.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ebc24305400, cur 1594836183 expire 1594836033 last 1594835956 [610841.642436] Lustre: fir-MDT0003: Connection restored to 7fc26a72-0608-4 (at 10.50.1.4@o2ib2) [610943.966331] Lustre: fir-MDT0003: Connection restored to a7c29338-fa1e-4 (at 10.50.17.13@o2ib2) [611036.246278] Lustre: fir-MDT0003: Connection restored to 1641a186-12ad-4 (at 10.50.1.7@o2ib2) [611036.254808] Lustre: Skipped 2 previous similar messages [611284.331061] Lustre: fir-MDT0003: Connection restored to 201c17c2-702e-4 (at 10.50.8.52@o2ib2) [611284.339674] Lustre: Skipped 2 previous similar messages [612103.350491] Lustre: fir-MDT0003: Connection restored to db3002b0-b461-4 (at 10.50.1.12@o2ib2) [612628.231250] Lustre: fir-MDT0003: haven't heard from client 02ea926c-6632-4 (at 10.50.14.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe9674400, cur 1594838042 expire 1594837892 last 1594837815 [612628.251221] Lustre: Skipped 2 previous similar messages [612704.228458] Lustre: fir-MDT0003: haven't heard from client 1b3e800b-d4ee-4 (at 10.50.6.10@o2ib2) in 160 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7890c00, cur 1594838118 expire 1594837968 last 1594837958 [612991.240537] Lustre: fir-MDT0003: haven't heard from client 6579911e-704e-4 (at 10.50.8.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbde467c00, cur 1594838405 expire 1594838255 last 1594838178 [613503.894054] Lustre: fir-MDT0003: Connection restored to 57c2e4ce-5ebf-4 (at 10.50.1.14@o2ib2) [613503.902668] Lustre: Skipped 1 previous similar message [613626.949931] Lustre: fir-MDT0003: Connection restored to 80fd9564-33c3-4 (at 10.50.5.61@o2ib2) [613626.958549] Lustre: Skipped 1 previous similar message [613844.261485] Lustre: fir-MDT0003: haven't heard from client e4f19b4d-4ac8-4 (at 10.50.5.63@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfebdec00, cur 1594839258 expire 1594839108 last 1594839031 [613980.990559] Lustre: fir-MDT0003: Connection restored to 02ea926c-6632-4 (at 10.50.14.7@o2ib2) [614309.276935] Lustre: fir-MDT0003: haven't heard from client 1a270eb0-1536-4 (at 10.50.13.14@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe82fa400, cur 1594839723 expire 1594839573 last 1594839496 [614385.284038] Lustre: fir-MDT0003: haven't heard from client abe8a499-d07c-4 (at 10.50.4.33@o2ib2) in 225 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe4a60000, cur 1594839799 expire 1594839649 last 1594839574 [615278.052557] Lustre: fir-MDT0003: Connection restored to 1b3e800b-d4ee-4 (at 10.50.6.10@o2ib2) [615383.036633] Lustre: fir-MDT0003: Connection restored to 6579911e-704e-4 (at 10.50.8.5@o2ib2) [616022.341365] Lustre: fir-MDT0003: Connection restored to 1a270eb0-1536-4 (at 10.50.13.14@o2ib2) [616022.350073] Lustre: Skipped 1 previous similar message [616218.615085] Lustre: fir-MDT0003: Connection restored to a5bf01f5-318d-4 (at 10.50.1.5@o2ib2) [617358.254584] Lustre: fir-MDT0003: Connection restored to e4f19b4d-4ac8-4 (at 10.50.5.63@o2ib2) [617395.297234] Lustre: fir-MDT0003: Connection restored to abe8a499-d07c-4 (at 10.50.4.33@o2ib2) [618754.402296] Lustre: fir-MDT0003: haven't heard from client d8d6e516-a270-4 (at 10.50.4.26@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf46fac00, cur 1594844168 expire 1594844018 last 1594843941 [618783.617148] Lustre: fir-MDT0003: Connection restored to 9da61e69-0e86-4 (at 10.50.1.29@o2ib2) [618783.625762] Lustre: Skipped 1 previous similar message [619438.419391] Lustre: fir-MDT0003: haven't heard from client 447b427e-4978-4 (at 10.50.8.67@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec747ffb000, cur 1594844852 expire 1594844702 last 1594844625 [619523.424752] Lustre: fir-MDT0003: Connection restored to b06eef0d-3006-4 (at 10.50.4.63@o2ib2) [619654.092024] Lustre: fir-MDT0003: Connection restored to a7736d92-3120-4 (at 10.50.3.41@o2ib2) [619665.207531] Lustre: fir-MDT0003: Connection restored to 69dff316-8c11-4 (at 10.50.8.67@o2ib2) [619675.732420] Lustre: fir-MDT0003: Connection restored to 6579911e-704e-4 (at 10.50.8.5@o2ib2) [619696.396851] Lustre: fir-MDT0003: Connection restored to 8ab56a74-861d-4 (at 10.50.9.13@o2ib2) [619734.479870] Lustre: fir-MDT0003: Connection restored to 6c909d6c-558d-4 (at 10.50.6.15@o2ib2) [619734.488490] Lustre: Skipped 2 previous similar messages [619769.493285] Lustre: fir-MDT0003: Connection restored to 541dff13-c9f2-4 (at 10.50.6.22@o2ib2) [619769.501903] Lustre: Skipped 2 previous similar messages [620902.394632] Lustre: fir-MDT0003: Connection restored to 65180038-e6c0-4 (at 10.50.8.71@o2ib2) [620998.297456] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [621069.467606] Lustre: fir-MDT0003: haven't heard from client de0edcdf-fb7f-4 (at 10.50.5.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba9e4000, cur 1594846483 expire 1594846333 last 1594846256 [621069.487512] Lustre: Skipped 9 previous similar messages [621255.471815] Lustre: fir-MDT0003: haven't heard from client a7a03a78-2bc3-4 (at 10.50.0.62@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbdfe1e400, cur 1594846669 expire 1594846519 last 1594846442 [621255.491810] Lustre: Skipped 3 previous similar messages [621293.921739] Lustre: fir-MDT0003: Connection restored to e757a848-a8d4-4 (at 10.50.2.69@o2ib2) [621293.930356] Lustre: Skipped 2 previous similar messages [622361.272381] Lustre: fir-MDT0003: Connection restored to 6d94aae5-de94-4 (at 10.50.1.37@o2ib2) [622393.743525] Lustre: fir-MDT0003: Connection restored to d7ec0e72-2343-4 (at 10.50.5.37@o2ib2) [622952.576607] Lustre: fir-MDT0003: Connection restored to f6f43919-59a9-4 (at 10.50.4.24@o2ib2) [623398.324119] Lustre: fir-MDT0003: Connection restored to 35a8e232-ad02-4 (at 10.50.5.60@o2ib2) [623719.653551] Lustre: fir-MDT0003: Connection restored to d8d6e516-a270-4 (at 10.50.4.26@o2ib2) [624032.736444] Lustre: fir-MDT0003: Connection restored to a7a03a78-2bc3-4 (at 10.50.0.62@o2ib2) [624157.551717] Lustre: fir-MDT0003: haven't heard from client 507c0a0c-c71b-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb6dbf6000, cur 1594849571 expire 1594849421 last 1594849344 [624295.858666] Lustre: fir-MDT0003: Connection restored to 7ed6edff-3703-4 (at 10.50.5.40@o2ib2) [624560.888790] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [624560.897497] Lustre: Skipped 1 previous similar message [625620.588941] Lustre: fir-MDT0003: haven't heard from client fdf8ee53-84fa-4 (at 10.50.14.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe7e4f000, cur 1594851034 expire 1594850884 last 1594850807 [625620.608929] Lustre: Skipped 1 previous similar message [626416.136668] Lustre: fir-MDT0003: Connection restored to 6686bc2c-9b8a-4 (at 10.50.2.29@o2ib2) [626433.135170] Lustre: fir-MDT0003: Client 3c54e12e-eaab-4 (at 10.49.23.13@o2ib1) reconnecting [627506.195160] Lustre: fir-MDT0003: Connection restored to fdf8ee53-84fa-4 (at 10.50.14.5@o2ib2) [627506.203775] Lustre: Skipped 1 previous similar message [628074.279317] Lustre: fir-MDT0003: Connection restored to 23f47988-82eb-4 (at 10.50.4.67@o2ib2) [628194.785889] Lustre: fir-MDT0003: Connection restored to 0451370c-7bf4-4 (at 10.50.8.22@o2ib2) [628291.661238] Lustre: fir-MDT0003: haven't heard from client 3880fcf4-dd28-4 (at 10.49.0.62@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf6bb8800, cur 1594853705 expire 1594853555 last 1594853478 [628372.813412] Lustre: fir-MDT0003: Connection restored to f3f10b94-326c-4 (at 10.50.8.23@o2ib2) [629353.953010] Lustre: fir-MDT0003: Connection restored to e5642ecc-5cff-4 (at 10.50.3.27@o2ib2) [630377.718150] Lustre: fir-MDT0003: haven't heard from client 2770e110-1884-4 (at 10.50.10.36@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe542d800, cur 1594855791 expire 1594855641 last 1594855564 [630453.720313] Lustre: fir-MDT0003: haven't heard from client b76d6209-d448-4 (at 10.50.9.53@o2ib2) in 218 seconds. I think it's dead, and I am evicting it. exp ffff8eebba683800, cur 1594855867 expire 1594855717 last 1594855649 [630453.740303] Lustre: Skipped 2 previous similar messages [630520.391598] Lustre: fir-MDT0003: Connection restored to 3880fcf4-dd28-4 (at 10.49.0.62@o2ib1) [630924.733041] Lustre: fir-MDT0003: haven't heard from client bc36cdd4-bb14-4 (at 10.50.8.69@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba571400, cur 1594856338 expire 1594856188 last 1594856111 [631000.735243] Lustre: fir-MDT0003: haven't heard from client ca04a123-cc10-4 (at 10.50.8.28@o2ib2) in 210 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe1525800, cur 1594856414 expire 1594856264 last 1594856204 [631000.755213] Lustre: Skipped 1 previous similar message [631257.670476] Lustre: fir-MDT0003: Connection restored to 732a55b6-1da1-4 (at 10.50.4.58@o2ib2) [632064.096977] Lustre: fir-MDT0003: Connection restored to c7236df2-950a-4 (at 10.50.3.39@o2ib2) [632257.037805] Lustre: fir-MDT0003: Connection restored to a4a911b8-4fa9-4 (at 10.50.7.70@o2ib2) [632433.677543] Lustre: fir-MDT0003: Connection restored to 7e4694aa-f47a-4 (at 10.50.4.28@o2ib2) [632679.462686] Lustre: fir-MDT0003: Connection restored to 2301850b-4dfd-4 (at 10.50.8.3@o2ib2) [632765.180307] Lustre: fir-MDT0003: Connection restored to cddeb248-e5b3-4 (at 10.50.8.31@o2ib2) [632808.992064] Lustre: fir-MDT0003: Connection restored to 5164c176-c47a-4 (at 10.50.6.34@o2ib2) [632829.446823] Lustre: fir-MDT0003: Connection restored to 2770e110-1884-4 (at 10.50.10.36@o2ib2) [632885.627080] Lustre: fir-MDT0003: Connection restored to 876bea2f-f254-4 (at 10.50.6.32@o2ib2) [632885.635702] Lustre: Skipped 1 previous similar message [632962.053516] Lustre: fir-MDT0003: Connection restored to 0da406f1-5bcc-4 (at 10.50.0.61@o2ib2) [632962.062131] Lustre: Skipped 1 previous similar message [633108.105303] Lustre: fir-MDT0003: Connection restored to 357fd555-de61-4 (at 10.50.6.33@o2ib2) [633108.113918] Lustre: Skipped 3 previous similar messages [633386.820502] Lustre: fir-MDT0003: Connection restored to 2d6d85ce-cbba-4 (at 10.50.2.42@o2ib2) [633386.829116] Lustre: Skipped 6 previous similar messages [633473.803210] Lustre: fir-MDT0003: haven't heard from client a7029368-92a4-4 (at 10.49.0.61@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe6088000, cur 1594858887 expire 1594858737 last 1594858660 [633907.743204] Lustre: fir-MDT0003: Connection restored to d56acfd9-fc73-4 (at 10.50.4.53@o2ib2) [633907.751820] Lustre: Skipped 12 previous similar messages [634568.850561] Lustre: fir-MDT0003: Connection restored to 5d1835d6-c34f-4 (at 10.50.2.37@o2ib2) [634568.859175] Lustre: Skipped 5 previous similar messages [635273.853423] Lustre: fir-MDT0003: Connection restored to 1c8b66d8-07b0-4 (at 10.50.1.56@o2ib2) [635273.862041] Lustre: Skipped 5 previous similar messages [635619.862493] Lustre: fir-MDT0003: haven't heard from client ba75cf72-2e52-4 (at 10.50.15.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf9a47400, cur 1594861033 expire 1594860883 last 1594860806 [636591.891239] Lustre: fir-MDT0003: haven't heard from client 0ce2bc6b-7111-4 (at 10.50.15.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe462d800, cur 1594862005 expire 1594861855 last 1594861778 [636935.682958] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [636935.691575] Lustre: Skipped 2 previous similar messages [637059.173102] Lustre: fir-MDT0003: Connection restored to ba75cf72-2e52-4 (at 10.50.15.10@o2ib2) [637059.181809] Lustre: Skipped 2 previous similar messages [637068.904990] Lustre: fir-MDT0003: haven't heard from client ca063ac0-974d-4 (at 10.50.5.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8dee3a000, cur 1594862482 expire 1594862332 last 1594862255 [637382.631051] Lustre: fir-MDT0003: Connection restored to 8ac2b9d2-d0c6-4 (at 10.50.10.21@o2ib2) [637789.220352] Lustre: fir-MDT0003: Connection restored to 1d56102f-edd8-4 (at 10.50.3.69@o2ib2) [638454.655617] Lustre: fir-MDT0003: Connection restored to 5b211e67-5ab6-4 (at 10.50.4.56@o2ib2) [638454.664234] Lustre: Skipped 2 previous similar messages [639796.067027] Lustre: fir-MDT0003: Connection restored to 3c68d909-9bf1-4 (at 10.50.4.60@o2ib2) [639796.075640] Lustre: Skipped 5 previous similar messages [639934.983590] Lustre: fir-MDT0003: haven't heard from client f636709d-6e8e-4 (at 10.50.15.11@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa691400, cur 1594865348 expire 1594865198 last 1594865121 [639935.003674] Lustre: Skipped 6 previous similar messages [641369.695565] Lustre: fir-MDT0003: Connection restored to f636709d-6e8e-4 (at 10.50.15.11@o2ib2) [641474.281224] Lustre: fir-MDT0003: Connection restored to a86aa23c-71a3-4 (at 10.50.8.56@o2ib2) [641812.649215] Lustre: fir-MDT0003: Connection restored to 76c8ecba-460b-4 (at 10.50.8.49@o2ib2) [642747.126692] Lustre: fir-MDT0003: Connection restored to 09c76335-fb75-4 (at 10.50.13.15@o2ib2) [643652.781131] Lustre: fir-MDT0003: Connection restored to 7b029440-65ce-4 (at 10.50.4.61@o2ib2) [643739.092350] Lustre: fir-MDT0003: haven't heard from client 40322d0f-19b4-4 (at 10.50.6.6@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ebc2405e800, cur 1594869152 expire 1594869002 last 1594868925 [644504.282080] Lustre: fir-MDT0003: Connection restored to 46ae5aee-7668-4 (at 10.50.4.36@o2ib2) [645823.152311] Lustre: fir-MDT0003: haven't heard from client 40cff0f9-258e-4 (at 10.50.3.48@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebda1d2c00, cur 1594871236 expire 1594871086 last 1594871009 [646246.218828] Lustre: fir-MDT0003: Connection restored to 40322d0f-19b4-4 (at 10.50.6.6@o2ib2) [647636.204568] Lustre: fir-MDT0003: haven't heard from client 6a3b989f-225f-4 (at 10.50.1.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebda5c8000, cur 1594873049 expire 1594872899 last 1594872822 [648267.077057] Lustre: fir-MDT0003: Connection restored to 40cff0f9-258e-4 (at 10.50.3.48@o2ib2) [650029.464659] Lustre: fir-MDT0003: Connection restored to 2878a559-521f-4 (at 10.50.8.42@o2ib2) [650068.637424] Lustre: fir-MDT0003: Connection restored to b4fdb3d4-da3a-4 (at 10.50.4.8@o2ib2) [650125.013672] Lustre: fir-MDT0003: Connection restored to ef158f0e-f25a-4 (at 10.50.4.5@o2ib2) [650133.491563] Lustre: fir-MDT0003: Connection restored to ed339afc-d01b-4 (at 10.50.4.7@o2ib2) [650165.151622] Lustre: fir-MDT0003: Connection restored to 7a5f87f4-a967-4 (at 10.50.8.46@o2ib2) [650203.675776] Lustre: fir-MDT0003: Connection restored to 8b91f98c-2384-4 (at 10.50.8.4@o2ib2) [650384.657315] Lustre: fir-MDT0003: Connection restored to d2e62dd1-f4b4-4 (at 10.50.4.6@o2ib2) [650855.501923] Lustre: fir-MDT0003: Connection restored to 6a3b989f-225f-4 (at 10.50.1.20@o2ib2) [651026.314363] Lustre: fir-MDT0003: haven't heard from client 97762868-29bb-4 (at 10.50.5.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa525800, cur 1594876439 expire 1594876289 last 1594876212 [651359.802013] Lustre: fir-MDT0003: Connection restored to 23328974-72ee-4 (at 10.50.8.1@o2ib2) [652054.403390] Lustre: fir-MDT0003: Connection restored to ad735b20-86f7-4 (at 10.50.3.35@o2ib2) [652054.412007] Lustre: Skipped 1 previous similar message [654208.534676] Lustre: fir-MDT0003: Connection restored to 97762868-29bb-4 (at 10.50.5.33@o2ib2) [654208.543296] Lustre: Skipped 1 previous similar message [657822.127974] Lustre: fir-MDT0003: Connection restored to 22fe755b-b0cf-4 (at 10.50.4.59@o2ib2) [661336.840545] Lustre: fir-MDT0003: Connection restored to c111cfdb-b78d-4 (at 10.50.4.72@o2ib2) [662594.471615] Lustre: fir-MDT0003: Connection restored to 99ec0974-0572-4 (at 10.50.8.2@o2ib2) [662673.228266] Lustre: fir-MDT0003: Connection restored to 6b899585-733a-4 (at 10.49.21.12@o2ib1) [662674.902396] Lustre: fir-MDT0003: Connection restored to 8cde9e8e-714a-4 (at 10.49.21.28@o2ib1) [662677.418592] Lustre: fir-MDT0003: Connection restored to e36fd89b-4f81-4 (at 10.50.8.13@o2ib2) [662691.255190] Lustre: fir-MDT0003: Connection restored to 8b633ae0-41d6-4 (at 10.49.21.9@o2ib1) [662700.795133] Lustre: fir-MDT0003: Connection restored to a66a1dfe-2b97-4 (at 10.49.21.10@o2ib1) [662700.803838] Lustre: Skipped 4 previous similar messages [663025.310735] Lustre: fir-MDT0003: Connection restored to c06ed6e0-33b3-4 (at 10.50.3.30@o2ib2) [663025.319356] Lustre: Skipped 8 previous similar messages [663344.015223] Lustre: fir-MDT0003: Connection restored to 87a11b69-7e43-4 (at 10.50.3.34@o2ib2) [663573.999319] Lustre: fir-MDT0003: Connection restored to 47d6ddf2-5bb0-4 (at 10.50.8.12@o2ib2) [663744.259507] Lustre: fir-MDT0003: Connection restored to de606222-0b76-4 (at 10.50.6.21@o2ib2) [665812.772511] Lustre: fir-MDT0003: Connection restored to 52ea266d-822e-4 (at 10.50.8.68@o2ib2) [666437.632854] Lustre: fir-MDT0003: Connection restored to b709eca5-b3c4-4 (at 10.50.5.19@o2ib2) [666899.715910] Lustre: fir-MDT0003: Connection restored to ab55037c-85a5-4 (at 10.50.13.8@o2ib2) [667054.798029] Lustre: fir-MDT0003: Connection restored to 7faeaa49-4ac3-4 (at 10.50.8.15@o2ib2) [668238.195426] Lustre: fir-MDT0003: Connection restored to b709eca5-b3c4-4 (at 10.50.5.19@o2ib2) [668694.833611] Lustre: fir-MDT0003: haven't heard from client 81fb710c-c1d3-4 (at 10.50.8.11@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9cd6400, cur 1594894107 expire 1594893957 last 1594893880 [668861.180663] Lustre: fir-MDT0003: Connection restored to dc0b824b-b8c1-4 (at 10.50.5.18@o2ib2) [669742.078205] Lustre: fir-MDT0003: Connection restored to 4ad1f496-f909-4 (at 10.50.8.10@o2ib2) [671610.865432] Lustre: fir-MDT0003: Connection restored to 81fb710c-c1d3-4 (at 10.50.8.11@o2ib2) [673268.158652] Lustre: fir-MDT0003: Connection restored to 9c96daf0-be54-4 (at 10.50.17.48@o2ib2) [673653.828635] Lustre: fir-MDT0003: Connection restored to 0666c367-bd86-4 (at 10.50.1.47@o2ib2) [675366.110184] Lustre: fir-MDT0003: Connection restored to 01ca646e-ef08-4 (at 10.50.3.18@o2ib2) [675659.607812] Lustre: fir-MDT0003: Connection restored to ea00688f-b441-4 (at 10.50.3.2@o2ib2) [677270.189322] Lustre: fir-MDT0003: Connection restored to 533362f3-c70b-4 (at 10.50.8.43@o2ib2) [677300.563301] Lustre: fir-MDT0003: Connection restored to 049d1506-ea1a-4 (at 10.50.1.61@o2ib2) [679781.172277] Lustre: fir-MDT0003: Connection restored to 68bc78fd-2409-4 (at 10.50.8.14@o2ib2) [691281.346923] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [691281.896934] Lustre: fir-MDT0003: Connection restored to f1b08221-f6fa-4 (at 10.50.5.7@o2ib2) [691281.905464] Lustre: Skipped 1 previous similar message [691392.521997] Lustre: fir-MDT0003: haven't heard from client c648cc29-d469-4 (at 10.50.5.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1ee672800, cur 1594916804 expire 1594916654 last 1594916577 [692700.187677] Lustre: fir-MDT0003: Connection restored to 5b40087f-9b2d-4 (at 10.50.7.68@o2ib2) [693015.539108] Lustre: fir-MDT0003: haven't heard from client e668636e-60ef-4 (at 10.50.3.30@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea9aefc800, cur 1594918427 expire 1594918277 last 1594918200 [693015.559076] Lustre: Skipped 3 previous similar messages [693058.852403] Lustre: fir-MDT0003: Connection restored to 011c361e-3203-4 (at 10.49.23.6@o2ib1) [693216.067418] Lustre: fir-MDT0003: Connection restored to 2301850b-4dfd-4 (at 10.50.8.3@o2ib2) [693220.212192] Lustre: fir-MDT0003: Connection restored to 2dcc403f-3684-4 (at 10.50.2.31@o2ib2) [693247.996003] Lustre: fir-MDT0003: Connection restored to 99ec0974-0572-4 (at 10.50.8.2@o2ib2) [693298.936127] Lustre: fir-MDT0003: Connection restored to a86aa23c-71a3-4 (at 10.50.8.56@o2ib2) [693347.662064] Lustre: fir-MDT0003: Connection restored to 599963ab-2ab0-4 (at 10.50.6.30@o2ib2) [693347.670683] Lustre: Skipped 3 previous similar messages [694783.978886] Lustre: fir-MDT0003: Connection restored to 5eaca4b2-8c5d-4 (at 10.50.8.16@o2ib2) [695740.557976] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [695741.100561] Lustre: fir-MDT0003: Connection restored to f1b08221-f6fa-4 (at 10.50.5.7@o2ib2) [695741.109091] Lustre: Skipped 1 previous similar message [695861.622310] Lustre: fir-MDT0003: haven't heard from client d25f2d90-487f-4 (at 10.50.5.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90bee9800, cur 1594921273 expire 1594921123 last 1594921046 [695861.642190] Lustre: Skipped 6 previous similar messages [696075.626306] Lustre: fir-MDT0003: haven't heard from client 5eda0247-3f40-4 (at 10.50.12.4@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfa295c00, cur 1594921487 expire 1594921337 last 1594921260 [696075.646279] Lustre: Skipped 2 previous similar messages [696151.629308] Lustre: fir-MDT0003: haven't heard from client bc329f61-28da-4 (at 10.49.27.35@o2ib1) in 225 seconds. I think it's dead, and I am evicting it. exp ffff8edbf2a9c400, cur 1594921563 expire 1594921413 last 1594921338 [696151.649365] Lustre: Skipped 6 previous similar messages [696482.642444] Lustre: fir-MDT0003: haven't heard from client c577e262-c6ca-4 (at 10.49.27.24@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf2a99000, cur 1594921894 expire 1594921744 last 1594921667 [696482.662501] Lustre: Skipped 4 previous similar messages [696920.676988] Lustre: fir-MDT0003: haven't heard from client 1ad60637-a7b3-4 (at 10.49.18.16@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9ec6800, cur 1594922332 expire 1594922182 last 1594922105 [697191.658803] Lustre: fir-MDT0003: haven't heard from client 7b56ef68-8822-4 (at 10.49.18.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb6bdd3800, cur 1594922603 expire 1594922453 last 1594922376 [697480.668747] Lustre: fir-MDT0003: haven't heard from client 77fda34a-8fac-4 (at 10.50.5.66@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe83bb000, cur 1594922892 expire 1594922742 last 1594922665 [697570.036875] Lustre: fir-MDT0003: Connection restored to c24f2361-7d8e-4 (at 10.50.12.11@o2ib2) [697578.848121] Lustre: fir-MDT0003: Connection restored to bb2f1c0f-bbbd-4 (at 10.50.16.2@o2ib2) [697787.638383] Lustre: fir-MDT0003: Connection restored to a5d9d05b-b770-4 (at 10.49.27.31@o2ib1) [697797.046015] Lustre: fir-MDT0003: Connection restored to d2220419-ed41-4 (at 10.49.27.2@o2ib1) [697804.931972] Lustre: fir-MDT0003: Connection restored to 67d3c325-62c3-4 (at 10.49.18.25@o2ib1) [697816.136574] Lustre: fir-MDT0003: Connection restored to 1bc2f5ee-2b10-4 (at 10.49.8.22@o2ib1) [697834.639062] Lustre: fir-MDT0003: Connection restored to f48b4fa2-2041-4 (at 10.49.7.14@o2ib1) [697834.647678] Lustre: Skipped 5 previous similar messages [697859.676749] Lustre: fir-MDT0003: haven't heard from client 3c924532-721f-4 (at 10.49.26.34@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf41da800, cur 1594923271 expire 1594923121 last 1594923044 [697859.696833] Lustre: Skipped 1 previous similar message [697868.711694] Lustre: fir-MDT0003: Connection restored to 6806bab1-57a2-4 (at 10.49.7.20@o2ib1) [697868.720316] Lustre: Skipped 14 previous similar messages [697932.736279] Lustre: fir-MDT0003: Connection restored to 3d2619c0-72f9-4 (at 10.49.18.33@o2ib1) [697932.744986] Lustre: Skipped 21 previous similar messages [697935.679911] Lustre: fir-MDT0003: haven't heard from client bc9d1238-8162-4 (at 10.49.27.19@o2ib1) in 165 seconds. I think it's dead, and I am evicting it. exp ffff8efbe6348c00, cur 1594923347 expire 1594923197 last 1594923182 [698078.434048] Lustre: fir-MDT0003: Connection restored to 82c2a3c8-72ea-4 (at 10.49.8.19@o2ib1) [698078.442666] Lustre: Skipped 14 previous similar messages [698557.122730] Lustre: fir-MDT0003: Connection restored to 57b64fe6-8f95-4 (at 10.50.16.16@o2ib2) [698557.131430] Lustre: Skipped 1 previous similar message [698664.701315] Lustre: fir-MDT0003: haven't heard from client 57dbd65b-1b6c-4 (at 10.50.2.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbf784d800, cur 1594924076 expire 1594923926 last 1594923849 [699342.916613] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [699342.925310] Lustre: Skipped 7 previous similar messages [699866.756085] Lustre: fir-MDT0003: haven't heard from client 98a68dea-1807-4 (at 10.49.18.12@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba6af400, cur 1594925278 expire 1594925128 last 1594925051 [700019.739102] Lustre: fir-MDT0003: haven't heard from client 3251b23c-c6df-4 (at 10.49.18.13@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe9677000, cur 1594925431 expire 1594925281 last 1594925204 [700565.753663] Lustre: fir-MDT0003: haven't heard from client 1987dd89-bc23-4 (at 10.49.27.18@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf5af2000, cur 1594925977 expire 1594925827 last 1594925750 [700680.858059] Lustre: fir-MDT0003: Connection restored to bbefacbf-0141-4 (at 10.50.5.65@o2ib2) [700680.866672] Lustre: Skipped 8 previous similar messages [700989.790757] Lustre: fir-MDT0003: Connection restored to 01a25c6f-3d8e-4 (at 10.50.16.15@o2ib2) [700989.799468] Lustre: Skipped 1 previous similar message [701121.770435] Lustre: fir-MDT0003: haven't heard from client b298491d-d606-4 (at 10.49.18.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba52a000, cur 1594926533 expire 1594926383 last 1594926306 [701422.239071] Lustre: fir-MDT0003: Connection restored to 3c924532-721f-4 (at 10.49.26.34@o2ib1) [701649.785361] Lustre: fir-MDT0003: haven't heard from client 67fe84a5-d3ea-4 (at 10.49.18.20@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe7328c00, cur 1594927061 expire 1594926911 last 1594926834 [701944.951116] Lustre: fir-MDT0003: Connection restored to 3f75082a-54e1-4 (at 10.49.27.17@o2ib1) [702700.340261] Lustre: fir-MDT0003: Connection restored to a9879d03-d73e-4 (at 10.49.27.32@o2ib1) [702700.348964] Lustre: Skipped 1 previous similar message [702831.818193] Lustre: fir-MDT0003: haven't heard from client 79b1fabc-0bf5-4 (at 10.50.16.16@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbfe880c00, cur 1594928243 expire 1594928093 last 1594928016 [702831.838252] Lustre: Skipped 1 previous similar message [702907.821941] Lustre: fir-MDT0003: haven't heard from client a229ccca-e5e6-4 (at 10.49.18.18@o2ib1) in 198 seconds. I think it's dead, and I am evicting it. exp ffff8eebf9ba1000, cur 1594928319 expire 1594928169 last 1594928121 [703381.761507] Lustre: fir-MDT0003: Connection restored to 67fe84a5-d3ea-4 (at 10.49.18.20@o2ib1) [703381.770212] Lustre: Skipped 1 previous similar message [703483.836555] Lustre: fir-MDT0003: haven't heard from client bba95675-41c9-4 (at 10.49.18.22@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbeb54dc00, cur 1594928895 expire 1594928745 last 1594928668 [703483.856611] Lustre: Skipped 1 previous similar message [704157.855504] Lustre: fir-MDT0003: haven't heard from client 9f0421a0-e4fb-4 (at 10.50.7.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb552cc800, cur 1594929569 expire 1594929419 last 1594929342 [704157.875382] Lustre: Skipped 1 previous similar message [704424.644995] Lustre: fir-MDT0003: Connection restored to 57b64fe6-8f95-4 (at 10.50.16.16@o2ib2) [704424.653699] Lustre: Skipped 4 previous similar messages [705172.724501] Lustre: fir-MDT0003: Connection restored to bba95675-41c9-4 (at 10.49.18.22@o2ib1) [705172.733210] Lustre: Skipped 2 previous similar messages [705281.886851] Lustre: fir-MDT0003: haven't heard from client edc187a0-d8e2-4 (at 10.49.18.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbef773400, cur 1594930693 expire 1594930543 last 1594930466 [705979.906300] Lustre: fir-MDT0003: haven't heard from client 0e1ba1cd-ecba-4 (at 10.50.3.49@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe455f400, cur 1594931391 expire 1594931241 last 1594931164 [706629.961485] Lustre: fir-MDT0003: Connection restored to 9f9bd3bb-4d2a-4 (at 10.50.7.1@o2ib2) [706629.970014] Lustre: Skipped 1 previous similar message [706999.858341] Lustre: fir-MDT0003: Connection restored to edc187a0-d8e2-4 (at 10.49.18.21@o2ib1) [707546.456887] Lustre: fir-MDT0003: Connection restored to 8b935c1c-1fdd-4 (at 10.49.7.19@o2ib1) [707896.565148] Lustre: fir-MDT0003: Connection restored to c5f20d9d-3b96-4 (at 10.50.1.17@o2ib2) [708167.967279] Lustre: fir-MDT0003: haven't heard from client dd6615bb-f7d8-4 (at 10.50.12.17@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf62a0800, cur 1594933579 expire 1594933429 last 1594933352 [708279.790414] LustreError: 26420:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'kousik' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H 2 kousik' to finish migration. [708388.973191] Lustre: fir-MDT0003: haven't heard from client a9deab29-fd4e-4 (at 10.49.7.19@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf6b76800, cur 1594933800 expire 1594933650 last 1594933573 [708514.066193] Lustre: fir-MDT0003: Connection restored to 8b935c1c-1fdd-4 (at 10.49.7.19@o2ib1) [708514.074812] Lustre: Skipped 3 previous similar messages [709652.742464] Lustre: fir-MDT0003: Connection restored to dd6615bb-f7d8-4 (at 10.50.12.17@o2ib2) [709652.751169] Lustre: Skipped 4 previous similar messages [710602.835253] Lustre: fir-MDT0003: Connection restored to 353ee429-3247-4 (at 10.49.27.16@o2ib1) [710602.843948] Lustre: Skipped 3 previous similar messages [711944.431138] Lustre: fir-MDT0003: Connection restored to 9a9645cd-cbe8-4 (at 10.49.27.21@o2ib1) [712020.074311] Lustre: fir-MDT0003: haven't heard from client 2a7be159-7447-4 (at 10.50.16.6@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf8a96c00, cur 1594937431 expire 1594937281 last 1594937204 [713994.747011] Lustre: fir-MDT0003: Connection restored to baf87720-4175-4 (at 10.49.18.26@o2ib1) [715210.376548] Lustre: fir-MDT0003: Connection restored to 0b52b3e2-3586-4 (at 10.50.5.17@o2ib2) [715674.571994] Lustre: fir-MDT0003: Connection restored to 2a7be159-7447-4 (at 10.50.16.6@o2ib2) [717866.233820] Lustre: fir-MDT0003: haven't heard from client ac2474cc-9d81-4 (at 10.50.10.35@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef85a311400, cur 1594943277 expire 1594943127 last 1594943050 [718801.692250] Lustre: fir-MDT0003: Connection restored to f58abbf9-25a6-4 (at 10.50.2.38@o2ib2) [720324.871963] Lustre: fir-MDT0003: Connection restored to 14563d49-fbad-4 (at 10.50.10.35@o2ib2) [720548.306913] Lustre: fir-MDT0003: haven't heard from client 9db36220-404c-4 (at 10.50.12.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba528400, cur 1594945959 expire 1594945809 last 1594945732 [720812.097662] Lustre: fir-MDT0003: Client b837e0e4-c6a2-4 (at 10.50.5.3@o2ib2) reconnecting [720812.105954] Lustre: fir-MDT0003: Connection restored to b879fe3b-ff02-4 (at 10.50.5.3@o2ib2) [720821.872546] Lustre: fir-MDT0003: Client e32faa4a-e19d-4 (at 10.50.5.72@o2ib2) reconnecting [720821.880926] Lustre: fir-MDT0003: Connection restored to f6e1116e-cdcb-4 (at 10.50.5.72@o2ib2) [721638.336871] Lustre: fir-MDT0003: haven't heard from client 3fb5c9b4-e930-4 (at 10.50.12.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf9a43800, cur 1594947049 expire 1594946899 last 1594946822 [722013.975618] Lustre: fir-MDT0003: Connection restored to c8623a46-739a-4 (at 10.50.8.9@o2ib2) [722951.377713] Lustre: fir-MDT0003: haven't heard from client b8f47b91-fce5-4 (at 10.50.12.16@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf4f7bc00, cur 1594948362 expire 1594948212 last 1594948135 [723000.235968] Lustre: fir-MDT0003: Connection restored to 3fb5c9b4-e930-4 (at 10.50.12.12@o2ib2) [723302.172078] Lustre: fir-MDT0003: Connection restored to 9db36220-404c-4 (at 10.50.12.1@o2ib2) [724348.366112] Lustre: fir-MDT0003: Connection restored to b8f47b91-fce5-4 (at 10.50.12.16@o2ib2) [724826.043461] Lustre: fir-MDT0003: Connection restored to 1674d3f1-717c-4 (at 10.50.12.13@o2ib2) [725183.454480] Lustre: fir-MDT0003: Connection restored to 3251b23c-c6df-4 (at 10.49.18.13@o2ib1) [727570.820204] Lustre: fir-MDT0003: Connection restored to 3a4bf8f7-8964-4 (at 10.50.3.19@o2ib2) [727571.410282] Lustre: fir-MDT0003: Connection restored to 139b52c8-b814-4 (at 10.50.3.23@o2ib2) [728697.534299] Lustre: fir-MDT0003: haven't heard from client 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf0faec00, cur 1594954108 expire 1594953958 last 1594953881 [728783.868101] Lustre: fir-MDT0003: Connection restored to aeb4a1df-673c-4 (at 10.49.26.36@o2ib1) [729368.554425] Lustre: fir-MDT0003: haven't heard from client 39b76fe5-e8ce-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf1397800, cur 1594954779 expire 1594954629 last 1594954552 [730129.297786] Lustre: fir-MDT0003: Connection restored to 996829be-6264-4 (at 10.49.27.11@o2ib1) [730272.048113] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [731057.234075] Lustre: fir-MDT0003: Connection restored to 12d1f44d-b724-4 (at 10.49.19.5@o2ib1) [733449.779054] Lustre: fir-MDT0003: Connection restored to aa1b179f-c256-4 (at 10.49.18.31@o2ib1) [737881.807561] Lustre: fir-MDT0003: haven't heard from client 23a507c8-c21d-4 (at 10.50.12.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe4548c00, cur 1594963292 expire 1594963142 last 1594963065 [737996.309359] Lustre: fir-MDT0003: Connection restored to 33766767-1c98-4 (at 10.50.12.2@o2ib2) [739616.363981] Lustre: fir-MDT0003: Connection restored to 23a507c8-c21d-4 (at 10.50.12.5@o2ib2) [741231.906389] Lustre: fir-MDT0003: haven't heard from client b5eaeda0-14b1-4 (at 10.50.12.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf4f7b000, cur 1594966642 expire 1594966492 last 1594966415 [742456.360656] Lustre: fir-MDT0003: Connection restored to b5eaeda0-14b1-4 (at 10.50.12.8@o2ib2) [743704.047105] Lustre: fir-MDT0003: Connection restored to 406e0eae-f761-4 (at 10.49.27.13@o2ib1) [744969.234558] Lustre: fir-MDT0003: Connection restored to 4cd2cb83-ac85-4 (at 10.49.27.23@o2ib1) [748364.458332] Lustre: fir-MDT0003: Connection restored to 66cfa3a8-5122-4 (at 10.49.22.20@o2ib1) [748936.955508] Lustre: fir-MDT0003: Connection restored to 3b740e15-b9b8-4 (at 10.49.23.5@o2ib1) [749283.141542] Lustre: fir-MDT0003: haven't heard from client b6cd88e9-552a-4 (at 10.50.16.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb3aa98800, cur 1594974693 expire 1594974543 last 1594974466 [749760.679965] Lustre: fir-MDT0003: Connection restored to a2ac76ed-a40d-4 (at 10.49.24.30@o2ib1) [749768.175229] Lustre: fir-MDT0003: Connection restored to 567e9ca5-e911-4 (at 10.49.25.3@o2ib1) [749769.845257] Lustre: fir-MDT0003: Connection restored to a40cc967-9003-4 (at 10.49.24.34@o2ib1) [749782.200058] Lustre: fir-MDT0003: Connection restored to 961ba1f6-aa96-4 (at 10.49.24.25@o2ib1) [749782.208759] Lustre: Skipped 1 previous similar message [749787.807025] Lustre: fir-MDT0003: Connection restored to f63521e5-3034-4 (at 10.49.24.32@o2ib1) [749795.952022] Lustre: fir-MDT0003: Connection restored to 2e7aa21e-50e5-4 (at 10.49.25.1@o2ib1) [749795.960658] Lustre: Skipped 4 previous similar messages [749814.374390] Lustre: fir-MDT0003: Connection restored to 26f086e5-fcd0-4 (at 10.49.24.29@o2ib1) [749814.383092] Lustre: Skipped 3 previous similar messages [750564.849976] Lustre: fir-MDT0003: Connection restored to d2c49c5b-480b-4 (at 10.50.16.3@o2ib2) [750564.858591] Lustre: Skipped 1 previous similar message [755762.428160] Lustre: fir-MDT0003: Connection restored to fc396b19-19df-4 (at 10.49.25.31@o2ib1) [755901.261516] Lustre: fir-MDT0003: Connection restored to 55c0bea7-1517-4 (at 10.49.25.29@o2ib1) [757550.397973] Lustre: fir-MDT0003: Connection restored to a7f92230-be1b-4 (at 10.49.31.10@o2ib1) [757595.843901] Lustre: fir-MDT0003: Connection restored to b3578d03-0ce7-4 (at 10.49.31.9@o2ib1) [757774.669997] Lustre: fir-MDT0003: Connection restored to 55c0bea7-1517-4 (at 10.49.25.29@o2ib1) [758257.812810] Lustre: fir-MDT0003: Connection restored to 8df7cec2-cd9e-4 (at 10.49.19.8@o2ib1) [760146.036071] Lustre: fir-MDT0003: Connection restored to 146f01e0-3559-4 (at 10.49.27.14@o2ib1) [760795.613675] Lustre: fir-MDT0003: Connection restored to 55049eb2-865a-4 (at 10.49.30.25@o2ib1) [760926.233037] Lustre: fir-MDT0003: Connection restored to b1083e0e-0d52-4 (at 10.49.30.26@o2ib1) [760968.975473] Lustre: fir-MDT0003: Connection restored to 47096ad4-880f-4 (at 10.49.30.30@o2ib1) [761437.739482] Lustre: fir-MDT0003: Connection restored to e00e0eca-5e15-4 (at 10.49.21.18@o2ib1) [762330.364235] Lustre: fir-MDT0003: Connection restored to 146f01e0-3559-4 (at 10.49.27.14@o2ib1) [764020.834078] Lustre: fir-MDT0003: Connection restored to 7cb9aeb5-8edb-4 (at 10.49.20.11@o2ib1) [764032.178610] Lustre: fir-MDT0003: Connection restored to a87eae0c-852c-4 (at 10.49.20.10@o2ib1) [764038.341699] Lustre: fir-MDT0003: Connection restored to b3b37d37-02a3-4 (at 10.49.20.5@o2ib1) [764041.097954] Lustre: fir-MDT0003: Connection restored to 3633d8d9-1faf-4 (at 10.49.20.12@o2ib1) [764244.469995] Lustre: fir-MDT0003: Connection restored to 168ce12d-371c-4 (at 10.49.20.2@o2ib1) [765877.466701] Lustre: fir-MDT0003: Connection restored to a87eae0c-852c-4 (at 10.49.20.10@o2ib1) [765879.468800] Lustre: fir-MDT0003: Connection restored to b3b37d37-02a3-4 (at 10.49.20.5@o2ib1) [766483.647068] Lustre: fir-MDT0003: haven't heard from client c38bd64d-068a-4 (at 10.49.29.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf62a2800, cur 1594991893 expire 1594991743 last 1594991666 [767825.059634] Lustre: fir-MDT0003: Connection restored to c38bd64d-068a-4 (at 10.49.29.3@o2ib1) [771679.723376] Lustre: fir-MDT0003: Connection restored to 86a1d03a-b5e3-4 (at 10.49.23.1@o2ib1) [771694.833180] Lustre: fir-MDT0003: Connection restored to a718fc49-81ef-4 (at 10.49.22.34@o2ib1) [771702.379550] Lustre: fir-MDT0003: Connection restored to 1c720c51-0f60-4 (at 10.49.22.36@o2ib1) [771711.092281] Lustre: fir-MDT0003: Connection restored to 72a402b6-8fa5-4 (at 10.49.30.28@o2ib1) [771711.100986] Lustre: Skipped 1 previous similar message [771720.364173] Lustre: fir-MDT0003: Connection restored to 4e98b7c7-8233-4 (at 10.49.22.33@o2ib1) [771720.372876] Lustre: Skipped 1 previous similar message [771742.198075] Lustre: fir-MDT0003: Connection restored to c6a6a6b3-27d1-4 (at 10.49.22.35@o2ib1) [771759.081140] Lustre: fir-MDT0003: Connection restored to 7f5aeee4-f3bd-4 (at 10.49.23.3@o2ib1) [772614.499145] Lustre: fir-MDT0003: Connection restored to cad16214-b868-4 (at 10.49.23.4@o2ib1) [772638.185191] Lustre: fir-MDT0003: Connection restored to afb92769-7f13-4 (at 10.49.30.29@o2ib1) [775236.903506] Lustre: fir-MDT0003: haven't heard from client 767be9ef-9ccb-4 (at 10.49.0.64@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd8f41400, cur 1595000646 expire 1595000496 last 1595000419 [778457.893203] Lustre: fir-MDT0003: Connection restored to 767be9ef-9ccb-4 (at 10.49.0.64@o2ib1) [779869.237183] Lustre: fir-MDT0003: Connection restored to 86d132c2-c1e9-4 (at 10.49.26.1@o2ib1) [782685.643735] Lustre: fir-MDT0003: Connection restored to ab20972d-17df-4 (at 10.49.20.7@o2ib1) [784578.436975] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [784589.166701] Lustre: fir-MDT0003: haven't heard from client 62272576-bfe1-4 (at 10.50.5.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edad4a51c00, cur 1595009998 expire 1595009848 last 1595009771 [784613.713298] Lustre: fir-MDT0003: Connection restored to f1b08221-f6fa-4 (at 10.50.5.7@o2ib2) [785102.153791] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [785120.190711] Lustre: fir-MDT0003: haven't heard from client 7eaad29f-874d-4 (at 10.50.5.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecab2d8c800, cur 1595010529 expire 1595010379 last 1595010302 [785120.210682] Lustre: Skipped 1 previous similar message [785160.450254] Lustre: fir-MDT0003: Connection restored to f1b08221-f6fa-4 (at 10.50.5.7@o2ib2) [785356.195719] Lustre: fir-MDT0003: haven't heard from client 933417ae-9ece-4 (at 10.50.3.66@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ebc24059000, cur 1595010765 expire 1595010615 last 1595010538 [785667.196662] Lustre: fir-MDT0003: haven't heard from client 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf4f7a000, cur 1595011076 expire 1595010926 last 1595010849 [786461.220849] Lustre: fir-MDT0003: haven't heard from client 3045cf85-ae54-4 (at 10.50.12.15@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf784b800, cur 1595011870 expire 1595011720 last 1595011643 [787584.210972] Lustre: fir-MDT0003: Connection restored to 08a3f471-0b5d-4 (at 10.50.3.63@o2ib2) [787588.150194] Lustre: fir-MDT0003: Connection restored to 208c0148-ed10-4 (at 10.50.3.51@o2ib2) [787606.519883] Lustre: fir-MDT0003: Connection restored to a56248c3-dd4e-4 (at 10.50.3.61@o2ib2) [787781.979087] Lustre: fir-MDT0003: Connection restored to 5907279d-e359-4 (at 10.50.3.57@o2ib2) [787790.046812] Lustre: fir-MDT0003: Connection restored to ee66905f-8719-4 (at 10.50.3.56@o2ib2) [787799.008077] Lustre: fir-MDT0003: Connection restored to 69ec2bd2-d3a8-4 (at 10.50.3.58@o2ib2) [787799.016697] Lustre: Skipped 1 previous similar message [787826.868081] Lustre: fir-MDT0003: Connection restored to c74dfabe-3d43-4 (at 10.50.3.52@o2ib2) [787826.876700] Lustre: Skipped 3 previous similar messages [787873.368394] Lustre: fir-MDT0003: Connection restored to 32a37e14-1a20-4 (at 10.50.3.60@o2ib2) [787873.377009] Lustre: Skipped 5 previous similar messages [788575.427399] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [789279.890902] Lustre: fir-MDT0003: Connection restored to 3ea20541-1680-4 (at 10.49.28.9@o2ib1) [795753.839907] Lustre: fir-MDT0003: Connection restored to 33766767-1c98-4 (at 10.50.12.2@o2ib2) [796595.500081] Lustre: fir-MDT0003: haven't heard from client 9e7b368a-b7fb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbf7849000, cur 1595022004 expire 1595021854 last 1595021777 [796633.761028] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [797295.522866] Lustre: fir-MDT0003: haven't heard from client f557bd1f-77dc-4 (at 10.49.26.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf2a98400, cur 1595022704 expire 1595022554 last 1595022477 [798498.009078] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [798537.557561] Lustre: fir-MDT0003: haven't heard from client dc13dd00-8ece-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbe95d2800, cur 1595023946 expire 1595023796 last 1595023719 [798591.126781] Lustre: fir-MDT0003: Connection restored to b21c67e4-c55c-4 (at 10.49.23.7@o2ib1) [798995.846736] Lustre: fir-MDT0003: Connection restored to f557bd1f-77dc-4 (at 10.49.26.21@o2ib1) [799223.079728] Lustre: fir-MDT0003: Connection restored to f8f91f2e-9101-4 (at 10.49.20.8@o2ib1) [799536.135112] Lustre: fir-MDT0003: Connection restored to ba253856-acf8-4 (at 10.49.20.4@o2ib1) [799821.670766] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [799861.587177] Lustre: fir-MDT0003: haven't heard from client c687d87a-fafd-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebdab28000, cur 1595025270 expire 1595025120 last 1595025043 [800087.105388] Lustre: fir-MDT0003: Connection restored to e2ce6895-b7e4-4 (at 10.49.24.14@o2ib1) [800475.173552] Lustre: fir-MDT0003: Connection restored to d43e2750-2f22-4 (at 10.50.4.14@o2ib2) [800475.182194] Lustre: Skipped 1 previous similar message [800787.472430] Lustre: fir-MDT0003: Connection restored to 7e117ff6-bb06-4 (at 10.50.4.13@o2ib2) [800787.481046] Lustre: Skipped 1 previous similar message [801029.618409] Lustre: fir-MDT0003: haven't heard from client ae557fb9-58e4-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf8844800, cur 1595026438 expire 1595026288 last 1595026211 [801098.956156] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [801098.964856] Lustre: Skipped 2 previous similar messages [801931.648312] Lustre: fir-MDT0003: haven't heard from client 6645b5aa-8d93-4 (at 10.50.3.51@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef2e2e8dc00, cur 1595027340 expire 1595027190 last 1595027113 [802583.157205] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [802618.666436] Lustre: fir-MDT0003: haven't heard from client a6f79715-a919-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef2d1b09800, cur 1595028027 expire 1595027877 last 1595027800 [802618.686511] Lustre: Skipped 2 previous similar messages [803335.513791] Lustre: fir-MDT0003: Connection restored to 08a3f471-0b5d-4 (at 10.50.3.63@o2ib2) [803374.354356] Lustre: fir-MDT0003: Connection restored to a56248c3-dd4e-4 (at 10.50.3.61@o2ib2) [803587.521928] Lustre: fir-MDT0003: Connection restored to 208c0148-ed10-4 (at 10.50.3.51@o2ib2) [804525.166275] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [804566.714147] Lustre: fir-MDT0003: haven't heard from client e84bcc9c-dd14-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb5260b800, cur 1595029975 expire 1595029825 last 1595029748 [805879.388631] Lustre: fir-MDT0003: Connection restored to 5f067692-7a56-4 (at 10.49.22.19@o2ib1) [805909.082262] Lustre: fir-MDT0003: Connection restored to 952e9894-a9c3-4 (at 10.49.22.32@o2ib1) [806350.270909] Lustre: fir-MDT0003: Connection restored to 596536a0-ce00-4 (at 10.49.24.7@o2ib1) [806357.657111] Lustre: fir-MDT0003: Connection restored to 42aa8e75-f8f8-4 (at 10.49.24.6@o2ib1) [806498.225993] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [806529.768615] Lustre: fir-MDT0003: haven't heard from client 62cd9827-017c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef7f9f4cc00, cur 1595031938 expire 1595031788 last 1595031711 [807832.118382] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [807865.802469] Lustre: fir-MDT0003: haven't heard from client ba526c2c-873c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edad8353c00, cur 1595033274 expire 1595033124 last 1595033047 [808199.853179] Lustre: fir-MDT0003: Connection restored to 33766767-1c98-4 (at 10.50.12.2@o2ib2) [808439.490015] Lustre: fir-MDT0003: Connection restored to 44106756-d9d6-4 (at 10.49.19.2@o2ib1) [808480.398049] Lustre: fir-MDT0003: Connection restored to 0a5b14a4-acde-4 (at 10.50.4.1@o2ib2) [809221.844722] Lustre: fir-MDT0003: haven't heard from client 59af13f4-37cc-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd434a800, cur 1595034630 expire 1595034480 last 1595034403 [809283.051230] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [810163.421752] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [810407.872907] Lustre: fir-MDT0003: haven't heard from client 412b445b-8964-4 (at 10.50.0.63@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf3cf1c00, cur 1595035816 expire 1595035666 last 1595035589 [811222.282816] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [811261.898557] Lustre: fir-MDT0003: haven't heard from client 2eadd25a-cd5d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1f6f67000, cur 1595036670 expire 1595036520 last 1595036443 [811261.918636] Lustre: Skipped 2 previous similar messages [811720.895471] Lustre: fir-MDT0003: Connection restored to 98a68dea-1807-4 (at 10.49.18.12@o2ib1) [811844.256077] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [812109.192068] Lustre: fir-MDT0003: Connection restored to a56248c3-dd4e-4 (at 10.50.3.61@o2ib2) [812182.263893] Lustre: fir-MDT0003: Connection restored to 08a3f471-0b5d-4 (at 10.50.3.63@o2ib2) [813455.904276] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [813483.955668] Lustre: fir-MDT0003: haven't heard from client 8709fad6-178f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef14b2d0400, cur 1595038892 expire 1595038742 last 1595038665 [813765.951094] Lustre: fir-MDT0003: Connection restored to 412b445b-8964-4 (at 10.50.0.63@o2ib2) [814500.526223] Lustre: fir-MDT0003: Connection restored to 7dadf4e0-5e2e-4 (at 10.50.2.41@o2ib2) [814790.259134] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [814822.992474] Lustre: fir-MDT0003: haven't heard from client 39799fd5-f741-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba9e0400, cur 1595040231 expire 1595040081 last 1595040004 [816987.051925] Lustre: fir-MDT0003: haven't heard from client f5d656e0-fdb9-4 (at 10.50.0.64@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf3cf2800, cur 1595042395 expire 1595042245 last 1595042168 [819197.115651] Lustre: fir-MDT0003: haven't heard from client 1cfc6bed-2088-4 (at 10.49.18.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfa291000, cur 1595044605 expire 1595044455 last 1595044378 [820309.063483] Lustre: fir-MDT0003: Connection restored to f5d656e0-fdb9-4 (at 10.50.0.64@o2ib2) [820907.997068] Lustre: fir-MDT0003: Connection restored to 1cfc6bed-2088-4 (at 10.49.18.23@o2ib1) [821055.171085] Lustre: fir-MDT0003: haven't heard from client 1dc77de6-25f0-4 (at 10.50.16.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe1663c00, cur 1595046463 expire 1595046313 last 1595046236 [824374.055728] Lustre: fir-MDT0003: Connection restored to a2e91212-e6f8-4 (at 10.50.16.5@o2ib2) [825146.293054] Lustre: fir-MDT0003: haven't heard from client 77a6e1a2-a880-4 (at 10.50.5.64@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf8bfb800, cur 1595050554 expire 1595050404 last 1595050327 [825662.997041] Lustre: fir-MDT0003: Connection restored to dc60fe8f-2472-4 (at 10.49.21.32@o2ib1) [827526.481620] Lustre: fir-MDT0003: Connection restored to 7667da4e-4b13-4 (at 10.49.22.13@o2ib1) [827546.862752] Lustre: fir-MDT0003: Connection restored to 6864c1d1-8c4a-4 (at 10.49.22.16@o2ib1) [827580.080620] Lustre: fir-MDT0003: Connection restored to 29a43597-a698-4 (at 10.49.22.17@o2ib1) [827589.460953] Lustre: fir-MDT0003: Connection restored to 628cb880-c9e9-4 (at 10.49.22.15@o2ib1) [828386.721931] Lustre: fir-MDT0003: Connection restored to 77a6e1a2-a880-4 (at 10.50.5.64@o2ib2) [832823.763720] Lustre: fir-MDT0003: Connection restored to 266c3980-550d-4 (at 10.49.27.15@o2ib1) [837072.780896] Lustre: fir-MDT0003: Connection restored to 3543637d-8998-4 (at 10.50.7.66@o2ib2) [844711.605194] Lustre: fir-MDT0003: Connection restored to 6904baaa-f9ea-4 (at 10.50.2.28@o2ib2) [854699.222235] Lustre: fir-MDT0003: Connection restored to 47f6645c-d953-4 (at 10.49.23.17@o2ib1) [854717.223760] Lustre: fir-MDT0003: Connection restored to 709d542b-7d67-4 (at 10.49.23.15@o2ib1) [854799.738955] Lustre: fir-MDT0003: Connection restored to a7fef789-dc59-4 (at 10.49.23.20@o2ib1) [854805.857558] Lustre: fir-MDT0003: Connection restored to 45e993a5-353a-4 (at 10.49.23.26@o2ib1) [854805.866273] Lustre: Skipped 1 previous similar message [854814.202155] Lustre: fir-MDT0003: Connection restored to b29c75f2-b005-4 (at 10.49.23.22@o2ib1) [854826.675707] Lustre: fir-MDT0003: Connection restored to da220933-ef8f-4 (at 10.49.23.27@o2ib1) [854826.684412] Lustre: Skipped 2 previous similar messages [854850.411048] Lustre: fir-MDT0003: Connection restored to d8f1cefc-f15a-4 (at 10.49.23.23@o2ib1) [854850.419748] Lustre: Skipped 3 previous similar messages [854889.244526] Lustre: fir-MDT0003: Connection restored to 3c54e12e-eaab-4 (at 10.49.23.13@o2ib1) [854889.253236] Lustre: Skipped 6 previous similar messages [855453.989090] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [857893.697170] Lustre: fir-MDT0003: Connection restored to e0f4ec1f-74f5-4 (at 10.50.17.45@o2ib2) [858893.539687] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [858945.270678] Lustre: fir-MDT0003: haven't heard from client 82f5842d-8daf-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea332aa000, cur 1595084352 expire 1595084202 last 1595084125 [860916.321838] Lustre: fir-MDT0003: Connection restored to e108b368-fe56-4 (at 10.49.20.9@o2ib1) [862613.979626] Lustre: fir-MDT0003: Connection restored to 799d62fb-f48e-4 (at 10.50.2.35@o2ib2) [868398.536720] Lustre: fir-MDT0003: haven't heard from client ea1cf07f-4048-4 (at 10.50.16.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf986c000, cur 1595093805 expire 1595093655 last 1595093578 [871433.153026] Lustre: fir-MDT0003: Connection restored to 84ec6789-095c-4 (at 10.50.3.11@o2ib2) [872028.839303] Lustre: fir-MDT0003: Connection restored to ea1cf07f-4048-4 (at 10.50.16.9@o2ib2) [872610.648210] Lustre: fir-MDT0003: Connection restored to b8f47b91-fce5-4 (at 10.50.12.16@o2ib2) [872645.654170] Lustre: fir-MDT0003: haven't heard from client 76dde559-0f5a-4 (at 10.50.12.16@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe7e48400, cur 1595098052 expire 1595097902 last 1595097825 [892682.730465] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [892693.191541] Lustre: fir-MDT0003: haven't heard from client a2b8bd57-731d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfe9b1800, cur 1595118099 expire 1595117949 last 1595117872 [898745.119980] Lustre: fir-MDT0003: Connection restored to e443e755-cf83-4 (at 10.49.31.8@o2ib1) [900244.460828] Lustre: fir-MDT0003: Connection restored to 5cbb2652-d662-4 (at 10.49.20.24@o2ib1) [900605.449591] Lustre: fir-MDT0003: Connection restored to e443e755-cf83-4 (at 10.49.31.8@o2ib1) [905158.770931] Lustre: fir-MDT0003: Connection restored to a65478d5-bd8b-4 (at 10.49.25.10@o2ib1) [905319.395749] Lustre: fir-MDT0003: Connection restored to e9ea1c99-2bbc-4 (at 10.49.25.12@o2ib1) [905328.845269] Lustre: fir-MDT0003: Connection restored to a06e0e49-d572-4 (at 10.49.25.11@o2ib1) [905328.853972] Lustre: Skipped 1 previous similar message [905337.047002] Lustre: fir-MDT0003: Connection restored to 25f70c45-7fcb-4 (at 10.49.25.30@o2ib1) [905365.391191] Lustre: fir-MDT0003: Connection restored to ceb475d4-9085-4 (at 10.49.25.32@o2ib1) [905382.127273] Lustre: fir-MDT0003: Connection restored to dc2dacaf-f9d1-4 (at 10.49.25.33@o2ib1) [905924.552454] Lustre: fir-MDT0003: haven't heard from client 77bd3e8d-f23a-4 (at 10.50.16.4@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbed670000, cur 1595131330 expire 1595131180 last 1595131103 [907198.385458] Lustre: fir-MDT0003: Connection restored to e9ea1c99-2bbc-4 (at 10.49.25.12@o2ib1) [909200.084152] Lustre: fir-MDT0003: Connection restored to 77bd3e8d-f23a-4 (at 10.50.16.4@o2ib2) [910031.423889] Lustre: fir-MDT0003: Connection restored to feb10c53-673e-4 (at 10.49.23.35@o2ib1) [910038.592201] Lustre: fir-MDT0003: Connection restored to 323da3bf-788c-4 (at 10.49.26.3@o2ib1) [910048.143452] Lustre: fir-MDT0003: Connection restored to 36db25a3-7b4a-4 (at 10.49.20.28@o2ib1) [910061.656166] Lustre: fir-MDT0003: Connection restored to b690dc1c-e96c-4 (at 10.49.23.34@o2ib1) [910090.246196] Lustre: fir-MDT0003: Connection restored to 2a0f99e6-78de-4 (at 10.49.20.25@o2ib1) [910098.494457] Lustre: fir-MDT0003: Connection restored to 77e9edf0-c242-4 (at 10.49.20.27@o2ib1) [923418.318359] Lustre: fir-MDT0003: Connection restored to 2bde024f-53e4-4 (at 10.49.24.20@o2ib1) [923554.016735] Lustre: fir-MDT0003: Connection restored to 3ce2d46c-9055-4 (at 10.49.25.22@o2ib1) [923574.764606] Lustre: fir-MDT0003: Connection restored to 296b6199-4dc5-4 (at 10.49.25.26@o2ib1) [923584.950491] Lustre: fir-MDT0003: Connection restored to 4ff256c9-2fbe-4 (at 10.49.24.19@o2ib1) [923609.998222] Lustre: fir-MDT0003: Connection restored to a86a6130-fbab-4 (at 10.49.25.24@o2ib1) [923610.006940] Lustre: Skipped 2 previous similar messages [925404.874615] Lustre: fir-MDT0003: Connection restored to 2bde024f-53e4-4 (at 10.49.24.20@o2ib1) [925404.883318] Lustre: Skipped 1 previous similar message [940173.260517] Lustre: fir-MDT0003: Connection restored to 8ae40cd4-4184-4 (at 10.50.17.47@o2ib2) [940173.269219] Lustre: Skipped 1 previous similar message [940504.560295] Lustre: fir-MDT0003: haven't heard from client 386836c6-da65-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaad6f0000, cur 1595165909 expire 1595165759 last 1595165682 [940539.472410] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [941280.383258] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [941336.584402] Lustre: fir-MDT0003: haven't heard from client caeddced-e259-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef3dafaa000, cur 1595166741 expire 1595166591 last 1595166514 [941496.736805] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [941545.606619] Lustre: fir-MDT0003: haven't heard from client e91262c7-f929-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed654fb8000, cur 1595166950 expire 1595166800 last 1595166723 [941709.045276] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [941786.598616] Lustre: fir-MDT0003: haven't heard from client 58c1380c-33ec-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed730a63400, cur 1595167191 expire 1595167041 last 1595166964 [942636.184449] Lustre: fir-MDT0003: Connection restored to 85be2a26-e86c-4 (at 10.49.25.16@o2ib1) [942641.825321] Lustre: fir-MDT0003: Connection restored to 8c732c76-1afd-4 (at 10.49.25.19@o2ib1) [942644.593483] Lustre: fir-MDT0003: Connection restored to 8a5439da-0a41-4 (at 10.49.25.20@o2ib1) [942658.181606] Lustre: fir-MDT0003: Connection restored to 28b22ccb-4ba9-4 (at 10.49.25.15@o2ib1) [942658.190313] Lustre: Skipped 1 previous similar message [942665.949377] Lustre: fir-MDT0003: Connection restored to f9805e39-5228-4 (at 10.49.25.14@o2ib1) [944499.865909] Lustre: fir-MDT0003: Connection restored to f9805e39-5228-4 (at 10.49.25.14@o2ib1) [944499.874612] Lustre: Skipped 2 previous similar messages [946803.976651] Lustre: fir-MDT0003: Connection restored to e3e2c99d-eb37-4 (at 10.49.30.27@o2ib1) [946954.940564] Lustre: fir-MDT0003: Connection restored to 9cd91d01-f2a9-4 (at 10.49.26.2@o2ib1) [946960.404328] Lustre: fir-MDT0003: Connection restored to 2498a273-7c0d-4 (at 10.49.25.9@o2ib1) [946964.965452] Lustre: fir-MDT0003: Connection restored to 49a8b992-15cd-4 (at 10.49.13.25@o2ib1) [947900.969215] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [947942.780539] Lustre: fir-MDT0003: haven't heard from client 72b2c2c9-a924-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea332af800, cur 1595173347 expire 1595173197 last 1595173120 [949384.576899] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [949440.820070] Lustre: fir-MDT0003: haven't heard from client 9feb57cc-ace4-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe6a64c00, cur 1595174845 expire 1595174695 last 1595174618 [973283.351412] Lustre: fir-MDT0003: Connection restored to 4e609ba2-d579-4 (at 10.49.23.8@o2ib1) [975473.875834] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [975498.556360] Lustre: fir-MDT0003: haven't heard from client f8eaf3d0-62a7-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe608d400, cur 1595200902 expire 1595200752 last 1595200675 [976045.571103] Lustre: fir-MDT0003: haven't heard from client c89c5d5d-f8cb-4 (at 10.49.20.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebc275a800, cur 1595201449 expire 1595201299 last 1595201222 [977744.941520] Lustre: fir-MDT0003: Connection restored to 1493a8e5-3349-4 (at 10.49.20.13@o2ib1) [977752.213146] Lustre: fir-MDT0003: Connection restored to 424b61db-3a7f-4 (at 10.49.20.18@o2ib1) [977753.398057] Lustre: fir-MDT0003: Connection restored to 994188cc-f96a-4 (at 10.49.20.15@o2ib1) [977762.498990] Lustre: fir-MDT0003: Connection restored to 21bb64c5-867e-4 (at 10.49.20.20@o2ib1) [977770.419085] Lustre: fir-MDT0003: Connection restored to db74c12a-97ee-4 (at 10.49.20.14@o2ib1) [977792.073483] Lustre: fir-MDT0003: Connection restored to c89c5d5d-f8cb-4 (at 10.49.20.17@o2ib1) [977792.082189] Lustre: Skipped 2 previous similar messages [979605.921168] Lustre: fir-MDT0003: Connection restored to db74c12a-97ee-4 (at 10.49.20.14@o2ib1) [979608.455652] Lustre: fir-MDT0003: Connection restored to 4b881965-7906-4 (at 10.49.20.16@o2ib1) [982641.828259] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [982666.749076] Lustre: fir-MDT0003: haven't heard from client 560c6504-6db6-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe455d400, cur 1595208070 expire 1595207920 last 1595207843 [982666.769133] Lustre: Skipped 4 previous similar messages [982985.136268] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [983020.777548] Lustre: fir-MDT0003: haven't heard from client 0db5adfd-e628-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef53456b800, cur 1595208424 expire 1595208274 last 1595208197 [984920.392655] Lustre: fir-MDT0003: Connection restored to 3bcbf823-3366-4 (at 10.49.24.10@o2ib1) [984941.199013] Lustre: fir-MDT0003: Connection restored to 1d1b0ba9-01b4-4 (at 10.49.24.13@o2ib1) [984955.653461] Lustre: fir-MDT0003: Connection restored to 03b1dc7b-7101-4 (at 10.49.24.11@o2ib1) [986850.654663] Lustre: fir-MDT0003: Connection restored to 9576d9e2-3e9e-4 (at 10.50.2.44@o2ib2) [986850.663275] Lustre: Skipped 1 previous similar message [1000625.265956] Lustre: fir-MDT0003: haven't heard from client 14a3ff29-f186-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd434a800, cur 1595226028 expire 1595225878 last 1595225801 [1000686.923005] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1000860.906537] Lustre: fir-MDT0003: Connection restored to f55a627e-8126-4 (at 10.49.24.2@o2ib1) [1000900.423969] Lustre: fir-MDT0003: Connection restored to 64304abb-d6ae-4 (at 10.49.24.1@o2ib1) [1000922.568765] Lustre: fir-MDT0003: Connection restored to 6667cc2f-8343-4 (at 10.49.24.3@o2ib1) [1000945.309801] Lustre: fir-MDT0003: Connection restored to 59b2d54c-b77e-4 (at 10.49.24.4@o2ib1) [1001284.748400] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1001316.286374] Lustre: fir-MDT0003: haven't heard from client 330556ad-1221-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb7f2a8800, cur 1595226719 expire 1595226569 last 1595226492 [1001676.032052] Lustre: fir-MDT0003: Connection restored to 124dcc1d-b08c-4 (at 10.49.24.5@o2ib1) [1002190.312052] Lustre: fir-MDT0003: haven't heard from client 8ba9858e-a75f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee717b9e800, cur 1595227593 expire 1595227443 last 1595227366 [1002235.666043] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1003589.266196] Lustre: fir-MDT0003: Connection restored to 124dcc1d-b08c-4 (at 10.49.24.5@o2ib1) [1003910.846092] Lustre: fir-MDT0003: Connection restored to fd317481-419c-4 (at 10.49.24.15@o2ib1) [1003932.749597] Lustre: fir-MDT0003: Connection restored to 1b4cf91c-2f47-4 (at 10.49.24.8@o2ib1) [1003932.758303] Lustre: Skipped 1 previous similar message [1005778.103903] Lustre: fir-MDT0003: Connection restored to 32364027-a6d2-4 (at 10.49.26.27@o2ib1) [1005778.112689] Lustre: Skipped 2 previous similar messages [1008704.249209] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1008710.503496] Lustre: fir-MDT0003: haven't heard from client 612706a8-7748-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee6a1984400, cur 1595234113 expire 1595233963 last 1595233886 [1015557.079024] Lustre: fir-MDT0003: Connection restored to 80f72b7b-87ed-4 (at 10.50.12.3@o2ib2) [1018718.794552] Lustre: fir-MDT0003: haven't heard from client ccda72ea-d635-4 (at 10.50.13.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbd6214400, cur 1595244121 expire 1595243971 last 1595243894 [1019908.128776] Lustre: fir-MDT0003: Connection restored to ccda72ea-d635-4 (at 10.50.13.1@o2ib2) [1024246.083293] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [1028604.436555] Lustre: 26720:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1028604.937897] Lustre: 26716:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1028604.949743] Lustre: 26716:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 588 previous similar messages [1028605.937594] Lustre: 26731:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1028605.949435] Lustre: 26731:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 903 previous similar messages [1028607.938285] Lustre: 26708:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1028607.950140] Lustre: 26708:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 1486 previous similar messages [1028611.938457] Lustre: 26707:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1028611.950317] Lustre: 26707:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 7085 previous similar messages [1035471.175708] perf: interrupt took too long (3968 > 3910), lowering kernel.perf_event_max_sample_rate to 50000 [1035524.708043] Lustre: fir-MDT0003: Connection restored to 4f59dcce-1d7f-4 (at 10.49.26.29@o2ib1) [1035640.957694] Lustre: fir-MDT0003: Connection restored to 72bb3b96-c0d6-4 (at 10.49.23.2@o2ib1) [1035653.097541] Lustre: fir-MDT0003: Connection restored to d210b337-47ca-4 (at 10.49.26.30@o2ib1) [1035653.106332] Lustre: Skipped 1 previous similar message [1035657.089195] Lustre: fir-MDT0003: Connection restored to 2efb017a-d814-4 (at 10.49.23.9@o2ib1) [1035668.229065] Lustre: fir-MDT0003: Connection restored to 204893ea-d32e-4 (at 10.49.23.11@o2ib1) [1035668.237858] Lustre: Skipped 1 previous similar message [1035680.900815] Lustre: fir-MDT0003: Connection restored to 24e68ce8-0c61-4 (at 10.49.24.17@o2ib1) [1035680.909616] Lustre: Skipped 2 previous similar messages [1035700.300398] Lustre: fir-MDT0003: Connection restored to 48fcfb30-eafb-4 (at 10.49.26.31@o2ib1) [1042211.463478] Lustre: fir-MDT0003: haven't heard from client 853865e9-6bca-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec5d6310c00, cur 1595267613 expire 1595267463 last 1595267386 [1042805.909643] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [1042805.918433] Lustre: Skipped 1 previous similar message [1043454.498501] Lustre: fir-MDT0003: haven't heard from client d72de17e-3802-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe6088000, cur 1595268856 expire 1595268706 last 1595268629 [1043489.355936] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1043993.514320] Lustre: fir-MDT0003: haven't heard from client df8ed271-2d39-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed94a270000, cur 1595269395 expire 1595269245 last 1595269168 [1044009.414697] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1044069.531435] Lustre: fir-MDT0003: haven't heard from client 415ee0e7-4aeb-4 (at 10.50.16.5@o2ib2) in 221 seconds. I think it's dead, and I am evicting it. exp ffff8eca7fedd000, cur 1595269471 expire 1595269321 last 1595269250 [1044069.551499] Lustre: Skipped 1 previous similar message [1044280.137492] Lustre: fir-MDT0003: Connection restored to 08a3f471-0b5d-4 (at 10.50.3.63@o2ib2) [1044360.857780] Lustre: fir-MDT0003: Connection restored to 671e3b0a-3ed9-4 (at 10.50.15.9@o2ib2) [1044371.982266] Lustre: fir-MDT0003: Connection restored to a2e91212-e6f8-4 (at 10.50.16.5@o2ib2) [1044424.450344] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1044488.526710] Lustre: fir-MDT0003: haven't heard from client 7fbf7948-7833-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf8630000, cur 1595269890 expire 1595269740 last 1595269663 [1044488.546866] Lustre: Skipped 3 previous similar messages [1044670.911339] Lustre: fir-MDT0003: Connection restored to 77bd3e8d-f23a-4 (at 10.50.16.4@o2ib2) [1044928.636374] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1044953.551550] Lustre: fir-MDT0003: haven't heard from client 42b491b4-d11c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebfeb71000, cur 1595270355 expire 1595270205 last 1595270128 [1045291.735828] Lustre: fir-MDT0003: Connection restored to 9fad833b-aecf-4 (at 10.50.3.59@o2ib2) [1045436.553509] Lustre: fir-MDT0003: haven't heard from client 0bc843a2-76e6-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90beebc00, cur 1595270838 expire 1595270688 last 1595270611 [1045827.164159] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1045827.172963] Lustre: Skipped 1 previous similar message [1045858.566377] Lustre: fir-MDT0003: haven't heard from client da8160e2-7a1f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd4349800, cur 1595271260 expire 1595271110 last 1595271033 [1046407.180202] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [1046506.584789] Lustre: fir-MDT0003: haven't heard from client 326abd9b-7b23-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa723400, cur 1595271908 expire 1595271758 last 1595271681 [1046967.597693] Lustre: fir-MDT0003: haven't heard from client 061beafe-0421-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb6d777800, cur 1595272369 expire 1595272219 last 1595272142 [1047159.672991] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [1047159.681697] Lustre: Skipped 2 previous similar messages [1047262.606153] Lustre: fir-MDT0003: haven't heard from client 0cf584c2-cbfc-4 (at 10.50.16.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec29fbd1c00, cur 1595272664 expire 1595272514 last 1595272437 [1047338.607171] Lustre: fir-MDT0003: haven't heard from client 29bf6dc9-7ec4-4 (at 10.49.26.35@o2ib1) in 196 seconds. I think it's dead, and I am evicting it. exp ffff8ef61cfe1c00, cur 1595272740 expire 1595272590 last 1595272544 [1047789.620988] Lustre: fir-MDT0003: haven't heard from client 80eab9aa-6739-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edac0e6e000, cur 1595273191 expire 1595273041 last 1595272964 [1048211.829199] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1048211.837988] Lustre: Skipped 2 previous similar messages [1048249.633720] Lustre: fir-MDT0003: haven't heard from client 3f979240-d508-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee6888d3400, cur 1595273651 expire 1595273501 last 1595273424 [1048665.644417] Lustre: fir-MDT0003: haven't heard from client e7446e09-0220-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda3debec00, cur 1595274067 expire 1595273917 last 1595273840 [1048989.179934] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1048989.188723] Lustre: Skipped 1 previous similar message [1049033.656200] Lustre: fir-MDT0003: haven't heard from client fba9210e-e6bb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb43b51400, cur 1595274435 expire 1595274285 last 1595274208 [1049813.150454] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1049813.159274] Lustre: Skipped 1 previous similar message [1049865.679221] Lustre: fir-MDT0003: haven't heard from client f7240801-a812-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed9aee5c400, cur 1595275267 expire 1595275117 last 1595275040 [1049865.699365] Lustre: Skipped 1 previous similar message [1050901.069346] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1050901.078141] Lustre: Skipped 1 previous similar message [1050928.709494] Lustre: fir-MDT0003: haven't heard from client b0341a2e-adb1-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef074489000, cur 1595276330 expire 1595276180 last 1595276103 [1050928.729658] Lustre: Skipped 1 previous similar message [1051682.268748] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1051682.277540] Lustre: Skipped 1 previous similar message [1051725.731945] Lustre: fir-MDT0003: haven't heard from client dfb515bb-7e72-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee7d3bf8000, cur 1595277127 expire 1595276977 last 1595276900 [1051725.752094] Lustre: Skipped 1 previous similar message [1052494.348180] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [1052494.356878] Lustre: Skipped 1 previous similar message [1052604.756670] Lustre: fir-MDT0003: haven't heard from client 5fc2cbf9-f384-4 (at 10.50.16.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef61cfe5400, cur 1595278006 expire 1595277856 last 1595277779 [1052604.776758] Lustre: Skipped 1 previous similar message [1053203.508613] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1053238.774398] Lustre: fir-MDT0003: haven't heard from client 3b0b76a0-da33-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8892fd400, cur 1595278640 expire 1595278490 last 1595278413 [1054874.628143] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [1054874.636845] Lustre: Skipped 1 previous similar message [1054993.524566] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1055037.825135] Lustre: fir-MDT0003: haven't heard from client 9b987ee6-62ba-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed9bd7e3800, cur 1595280439 expire 1595280289 last 1595280212 [1055037.845285] Lustre: Skipped 2 previous similar messages [1055226.829731] Lustre: fir-MDT0003: haven't heard from client 7dfea560-12b5-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeba7eee400, cur 1595280628 expire 1595280478 last 1595280401 [1055427.252839] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1055447.835652] Lustre: fir-MDT0003: haven't heard from client d509ca4a-57f2-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf98f7000, cur 1595280849 expire 1595280699 last 1595280622 [1059788.927770] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1059788.936559] Lustre: Skipped 1 previous similar message [1059819.955610] Lustre: fir-MDT0003: haven't heard from client f5a9c268-4c0c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1575a7000, cur 1595285221 expire 1595285071 last 1595284994 [1061080.030317] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1061120.991271] Lustre: fir-MDT0003: haven't heard from client 1cbda97c-92b6-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed4cef26400, cur 1595286522 expire 1595286372 last 1595286295 [1061187.196296] Lustre: fir-MDT0003: Client 6a159c0f-6ed7-4 (at 10.49.19.3@o2ib1) reconnecting [1061187.204820] Lustre: fir-MDT0003: Connection restored to c1151843-0075-4 (at 10.49.19.3@o2ib1) [1061942.255800] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1061986.014921] Lustre: fir-MDT0003: haven't heard from client 1a412aaa-842c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edec9fda000, cur 1595287387 expire 1595287237 last 1595287160 [1062979.674450] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1062998.041416] Lustre: fir-MDT0003: haven't heard from client d6a11b17-ac01-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef534569800, cur 1595288399 expire 1595288249 last 1595288172 [1063883.205714] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1063909.066949] Lustre: fir-MDT0003: haven't heard from client b1b06708-e68d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee0dea01800, cur 1595289310 expire 1595289160 last 1595289083 [1065008.095819] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1065040.097618] Lustre: fir-MDT0003: haven't heard from client 912fe622-dd7b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaa4613800, cur 1595290441 expire 1595290291 last 1595290214 [1065619.718149] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1065663.114485] Lustre: fir-MDT0003: haven't heard from client b8bdea20-f502-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec916538000, cur 1595291064 expire 1595290914 last 1595290837 [1066357.386349] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1066392.134300] Lustre: fir-MDT0003: haven't heard from client 8cfa1b8a-536d-4 (at 10.50.16.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8fbe10c00, cur 1595291793 expire 1595291643 last 1595291566 [1067231.009651] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1067262.157729] Lustre: fir-MDT0003: haven't heard from client 6ffd0ac4-ffdb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaa4611000, cur 1595292663 expire 1595292513 last 1595292436 [1067262.177903] Lustre: Skipped 1 previous similar message [1068274.828639] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [1068680.856843] Lustre: 26730:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1068680.868669] Lustre: 26730:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 2325 previous similar messages [1068681.856541] Lustre: 26734:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [1068681.868375] Lustre: 26734:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 1393 previous similar messages [1070510.246756] Lustre: fir-MDT0003: haven't heard from client ceff0355-0290-4 (at 10.50.16.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec5b4ed9800, cur 1595295911 expire 1595295761 last 1595295684 [1071768.764508] Lustre: fir-MDT0003: Connection restored to 5d61e1c4-cb41-4 (at 10.50.8.54@o2ib2) [1073007.965996] Lustre: fir-MDT0003: Connection restored to 6b358b4f-23e3-4 (at 10.50.16.1@o2ib2) [1073775.902923] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1073806.339601] Lustre: fir-MDT0003: haven't heard from client f51032ba-50cb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed94a275000, cur 1595299207 expire 1595299057 last 1595298980 [1078943.610324] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1078971.492717] Lustre: fir-MDT0003: haven't heard from client 991f6922-ae5f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd8e57800, cur 1595304372 expire 1595304222 last 1595304145 [1086098.891991] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1086121.697598] Lustre: fir-MDT0003: haven't heard from client 83a7bb66-1a17-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb6c50d800, cur 1595311522 expire 1595311372 last 1595311295 [1086559.992401] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1086602.712283] Lustre: fir-MDT0003: haven't heard from client 7afa044e-16d6-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfebdd000, cur 1595312003 expire 1595311853 last 1595311776 [1088802.798284] Lustre: fir-MDT0003: Connection restored to 1c060d97-c2ff-4 (at 10.50.9.39@o2ib2) [1088808.775645] Lustre: fir-MDT0003: haven't heard from client 89def38d-8c5d-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe2e15000, cur 1595314209 expire 1595314059 last 1595313982 [1090602.223109] Lustre: fir-MDT0003: Connection restored to a8b8c8c3-d34e-4 (at 10.50.17.16@o2ib2) [1090618.828704] Lustre: fir-MDT0003: haven't heard from client a2bf8ae4-2b64-4 (at 10.50.17.16@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8fbe7f800, cur 1595316019 expire 1595315869 last 1595315792 [1094631.497916] Lustre: fir-MDT0003: Connection restored to eddbd793-d791-4 (at 10.49.25.23@o2ib1) [1102869.180249] Lustre: fir-MDT0003: haven't heard from client a454fbf7-dba0-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa720c00, cur 1595328269 expire 1595328119 last 1595328042 [1102902.445041] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1103670.030515] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1103707.204291] Lustre: fir-MDT0003: haven't heard from client f2cfb129-a315-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef2d1b0cc00, cur 1595329107 expire 1595328957 last 1595328880 [1105077.243655] Lustre: fir-MDT0003: haven't heard from client c62f043f-a78f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea35b95c00, cur 1595330477 expire 1595330327 last 1595330250 [1105149.809761] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1105679.260849] Lustre: fir-MDT0003: haven't heard from client 14e5e58e-2d76-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea4d703000, cur 1595331079 expire 1595330929 last 1595330852 [1105755.130684] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1106480.496978] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1106498.559637] Lustre: fir-MDT0003: Connection restored to 7bb94634-c43c-4 (at 10.50.8.38@o2ib2) [1106510.284545] Lustre: fir-MDT0003: haven't heard from client 565e3323-2174-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaa16b8000, cur 1595331910 expire 1595331760 last 1595331683 [1107703.962938] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1107737.319472] Lustre: fir-MDT0003: haven't heard from client 56eaf931-8c7c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee9c1ab3c00, cur 1595333137 expire 1595332987 last 1595332910 [1108477.710459] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1108509.341507] Lustre: fir-MDT0003: haven't heard from client 4e5d20fe-f82f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebc275c400, cur 1595333909 expire 1595333759 last 1595333682 [1109508.371104] Lustre: fir-MDT0003: haven't heard from client 875a2281-bf4b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea4d704400, cur 1595334908 expire 1595334758 last 1595334681 [1109546.264261] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1110141.364442] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1110175.389570] Lustre: fir-MDT0003: haven't heard from client 36d24a72-715c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba529400, cur 1595335575 expire 1595335425 last 1595335348 [1112577.458243] Lustre: fir-MDT0003: haven't heard from client a1100a56-dd3d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ede73e6b800, cur 1595337977 expire 1595337827 last 1595337750 [1112613.921758] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1113142.661617] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1113168.476545] Lustre: fir-MDT0003: haven't heard from client b10e6c33-4ebd-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8fbe11c00, cur 1595338568 expire 1595338418 last 1595338341 [1114389.511875] Lustre: fir-MDT0003: haven't heard from client 7b287724-3d11-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbddade800, cur 1595339789 expire 1595339639 last 1595339562 [1114415.465086] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1114878.432902] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1114919.527149] Lustre: fir-MDT0003: haven't heard from client ae21d59d-1af2-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee5d87f2000, cur 1595340319 expire 1595340169 last 1595340092 [1116698.578634] Lustre: fir-MDT0003: haven't heard from client 092c2b8e-484b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed81850c800, cur 1595342098 expire 1595341948 last 1595341871 [1116754.732332] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1117225.683179] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1117269.595004] Lustre: fir-MDT0003: haven't heard from client 18fabf15-6370-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed7dfa91000, cur 1595342669 expire 1595342519 last 1595342442 [1118103.619202] Lustre: fir-MDT0003: haven't heard from client 3e94a754-5850-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef2e2e8e400, cur 1595343503 expire 1595343353 last 1595343276 [1118144.466705] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1118763.924669] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1118809.639072] Lustre: fir-MDT0003: haven't heard from client 497064fd-e210-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef85a313400, cur 1595344209 expire 1595344059 last 1595343982 [1119711.664765] Lustre: fir-MDT0003: haven't heard from client ade2c484-e872-4 (at 10.50.8.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe12db400, cur 1595345111 expire 1595344961 last 1595344884 [1120039.253426] Lustre: fir-MDT0003: Connection restored to 5d61e1c4-cb41-4 (at 10.50.8.54@o2ib2) [1121462.854892] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1121498.714993] Lustre: fir-MDT0003: haven't heard from client 760b07b7-554d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea332ab400, cur 1595346898 expire 1595346748 last 1595346671 [1132397.405679] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1132402.024308] Lustre: fir-MDT0003: haven't heard from client 4b8ec38f-a58c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7883c00, cur 1595357801 expire 1595357651 last 1595357574 [1132757.004607] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1132801.035658] Lustre: fir-MDT0003: haven't heard from client a35a7489-86de-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb26aa3400, cur 1595358200 expire 1595358050 last 1595357973 [1133091.464348] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1133136.045017] Lustre: fir-MDT0003: haven't heard from client 6fc86f78-1b4f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd9365c00, cur 1595358535 expire 1595358385 last 1595358308 [1133485.854729] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1133520.054729] Lustre: fir-MDT0003: haven't heard from client cef4b2a7-c87a-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee8fcfc4c00, cur 1595358919 expire 1595358769 last 1595358692 [1133826.451536] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1133890.065128] Lustre: fir-MDT0003: haven't heard from client 217f86db-20c6-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90beee000, cur 1595359289 expire 1595359139 last 1595359062 [1134203.758762] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1134230.075171] Lustre: fir-MDT0003: haven't heard from client 573a407f-c7d7-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed4cef25000, cur 1595359629 expire 1595359479 last 1595359402 [1134568.095628] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1134607.086223] Lustre: fir-MDT0003: haven't heard from client bcb38b30-ea2a-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf786d800, cur 1595360006 expire 1595359856 last 1595359779 [1134952.405530] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1134997.097356] Lustre: fir-MDT0003: haven't heard from client 7ae190f6-c11a-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed5b436e000, cur 1595360396 expire 1595360246 last 1595360169 [1134997.117513] Lustre: Skipped 1 previous similar message [1135452.967874] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1135482.111117] Lustre: fir-MDT0003: haven't heard from client 941b5e25-b317-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef7fb25f800, cur 1595360881 expire 1595360731 last 1595360654 [1135887.642294] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1135932.123940] Lustre: fir-MDT0003: haven't heard from client 5542a090-c068-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed654fb8000, cur 1595361331 expire 1595361181 last 1595361104 [1136365.242100] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1136392.136679] Lustre: fir-MDT0003: haven't heard from client 0373056b-d53d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed81850f000, cur 1595361791 expire 1595361641 last 1595361564 [1137172.987825] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1137172.996614] Lustre: Skipped 1 previous similar message [1137216.165261] Lustre: fir-MDT0003: haven't heard from client 513d4bca-cde4-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef85a311000, cur 1595362615 expire 1595362465 last 1595362388 [1137216.185400] Lustre: Skipped 1 previous similar message [1138761.443204] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1138761.452018] Lustre: Skipped 1 previous similar message [1138789.204195] Lustre: fir-MDT0003: haven't heard from client 6288ba95-e195-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef560115000, cur 1595364188 expire 1595364038 last 1595363961 [1138789.224337] Lustre: Skipped 1 previous similar message [1139635.332142] Lustre: fir-MDT0003: Connection restored to 591c5fe0-0c38-4 (at 10.49.26.7@o2ib1) [1139659.553921] Lustre: fir-MDT0003: Connection restored to eb610fb1-89b6-4 (at 10.49.26.8@o2ib1) [1139776.081920] Lustre: fir-MDT0003: Connection restored to 6dcc382e-aa25-4 (at 10.49.26.6@o2ib1) [1139776.090629] Lustre: Skipped 1 previous similar message [1140445.250623] Lustre: fir-MDT0003: haven't heard from client 11b80835-2d2f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee8cc35c400, cur 1595365844 expire 1595365694 last 1595365617 [1140447.455709] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1140790.978095] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1140850.262036] Lustre: fir-MDT0003: haven't heard from client 16d0406e-5757-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee6dfb18400, cur 1595366249 expire 1595366099 last 1595366022 [1141195.604179] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1141220.272745] Lustre: fir-MDT0003: haven't heard from client 64b510d4-6b6b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef5272f4c00, cur 1595366619 expire 1595366469 last 1595366392 [1141693.783142] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1141750.287145] Lustre: fir-MDT0003: haven't heard from client 20b89803-7d65-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebda5d5000, cur 1595367149 expire 1595366999 last 1595366922 [1142062.300693] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1142122.297771] Lustre: fir-MDT0003: haven't heard from client e0e91e9d-4918-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee9c1ab6c00, cur 1595367521 expire 1595367371 last 1595367294 [1142498.143590] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1142540.309400] Lustre: fir-MDT0003: haven't heard from client eb11c3e6-f067-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1b2e5e000, cur 1595367939 expire 1595367789 last 1595367712 [1142939.187928] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1142977.320619] Lustre: fir-MDT0003: haven't heard from client b7b621e9-6bdb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaad6f6800, cur 1595368376 expire 1595368226 last 1595368149 [1143413.891915] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1143842.345472] Lustre: fir-MDT0003: haven't heard from client cb91fc14-691b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf315f400, cur 1595369241 expire 1595369091 last 1595369014 [1143842.365615] Lustre: Skipped 1 previous similar message [1144265.896763] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1144265.905554] Lustre: Skipped 1 previous similar message [1144644.367978] Lustre: fir-MDT0003: haven't heard from client d1852e51-1926-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb6dbd2c00, cur 1595370043 expire 1595369893 last 1595369816 [1144644.388163] Lustre: Skipped 1 previous similar message [1144908.321062] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1144908.329851] Lustre: Skipped 1 previous similar message [1145287.385214] Lustre: fir-MDT0003: haven't heard from client 44cfa934-e1bb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee7193c7400, cur 1595370686 expire 1595370536 last 1595370459 [1145287.405359] Lustre: Skipped 1 previous similar message [1145588.786292] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1145588.795087] Lustre: Skipped 1 previous similar message [1145942.403251] Lustre: fir-MDT0003: haven't heard from client 092dad5a-12cd-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee790e1e400, cur 1595371341 expire 1595371191 last 1595371114 [1145942.423406] Lustre: Skipped 1 previous similar message [1146239.120920] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1146239.129714] Lustre: Skipped 1 previous similar message [1146718.424634] Lustre: fir-MDT0003: haven't heard from client 31b84097-1f1a-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee61bf64800, cur 1595372117 expire 1595371967 last 1595371890 [1146718.444778] Lustre: Skipped 1 previous similar message [1146914.817037] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [1146914.825825] Lustre: Skipped 1 previous similar message [1147586.785458] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [1147586.794253] Lustre: Skipped 2 previous similar messages [1147643.452216] Lustre: fir-MDT0003: haven't heard from client 9972b6ec-24b1-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaad6f0400, cur 1595373042 expire 1595372892 last 1595372815 [1147643.472361] Lustre: Skipped 3 previous similar messages [1148295.288399] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [1148295.297189] Lustre: Skipped 4 previous similar messages [1148334.468648] Lustre: fir-MDT0003: haven't heard from client cf50ed0e-8099-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edccaf76800, cur 1595373733 expire 1595373583 last 1595373506 [1148334.488821] Lustre: Skipped 4 previous similar messages [1148921.253312] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [1148921.262111] Lustre: Skipped 4 previous similar messages [1148985.485940] Lustre: fir-MDT0003: haven't heard from client 3e1b1ccf-fe20-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed7dfa96c00, cur 1595374384 expire 1595374234 last 1595374157 [1148985.506102] Lustre: Skipped 4 previous similar messages [1149634.391771] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1149634.400566] Lustre: Skipped 2 previous similar messages [1149658.504057] Lustre: fir-MDT0003: haven't heard from client dfec2c25-64ec-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda52f0b800, cur 1595375057 expire 1595374907 last 1595374830 [1149658.524203] Lustre: Skipped 2 previous similar messages [1150501.563927] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1150501.572729] Lustre: Skipped 2 previous similar messages [1150553.528033] Lustre: fir-MDT0003: haven't heard from client 8951c0f8-d02d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee82cbf2c00, cur 1595375952 expire 1595375802 last 1595375725 [1150553.548206] Lustre: Skipped 2 previous similar messages [1151197.684533] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1151197.693324] Lustre: Skipped 1 previous similar message [1151254.545972] Lustre: fir-MDT0003: haven't heard from client eb0f402b-a688-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaad6f6c00, cur 1595376653 expire 1595376503 last 1595376426 [1151254.566145] Lustre: Skipped 1 previous similar message [1151838.643576] Lustre: fir-MDT0003: Connection restored to 8df17f75-b2f1-4 (at 10.50.16.11@o2ib2) [1153941.619770] Lustre: fir-MDT0003: haven't heard from client eed930ae-77f6-4 (at 10.49.27.27@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb7f2abc00, cur 1595379340 expire 1595379190 last 1595379113 [1154023.015393] Lustre: fir-MDT0003: Connection restored to 9c11b6d7-9d47-4 (at 10.49.27.27@o2ib1) [1155029.649948] Lustre: fir-MDT0003: haven't heard from client 8caf45d9-f169-4 (at 10.49.20.31@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf5af0c00, cur 1595380428 expire 1595380278 last 1595380201 [1156595.651565] Lustre: fir-MDT0003: Connection restored to 5b135430-c01e-4 (at 10.49.20.32@o2ib1) [1156720.255779] Lustre: fir-MDT0003: Connection restored to 777d00ac-6447-4 (at 10.49.20.23@o2ib1) [1156741.777113] Lustre: fir-MDT0003: Connection restored to be76c0eb-7dd8-4 (at 10.49.20.6@o2ib1) [1158312.369493] Lustre: fir-MDT0003: Connection restored to 13246d3f-4765-4 (at 10.49.26.12@o2ib1) [1158312.378278] Lustre: Skipped 3 previous similar messages [1158322.723915] Lustre: fir-MDT0003: Connection restored to 04377607-a6c6-4 (at 10.49.26.9@o2ib1) [1158332.320668] Lustre: fir-MDT0003: Connection restored to e3a520dd-3511-4 (at 10.49.26.11@o2ib1) [1158601.910372] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1158601.919167] Lustre: Skipped 1 previous similar message [1158626.749354] Lustre: fir-MDT0003: haven't heard from client d088cab8-2048-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaad6f3000, cur 1595384025 expire 1595383875 last 1595383798 [1158626.769514] Lustre: Skipped 1 previous similar message [1165705.700004] LustreError: 24463:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-yNsjQCbR/BUILD/lustre-2.12.5_RC1/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1196874.908158] Lustre: 23961:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595422263/real 1595422263] req@ffff8ec58137cc80 x1671791662156416/t0(0) o41->fir-MDT0002-osp-MDT0003@10.0.10.53@o2ib7:24/4 lens 224/368 e 0 to 1 dl 1595422270 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1196874.936462] Lustre: fir-MDT0002-osp-MDT0003: Connection to fir-MDT0002 (at 10.0.10.53@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1196894.986297] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.49.24.22@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1196902.987396] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.8.51@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1196903.004866] LustreError: Skipped 398 previous similar messages [1196919.020650] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.5.10@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1196919.038110] LustreError: Skipped 908 previous similar messages [1196954.648805] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.7.50@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1196954.666267] LustreError: Skipped 142 previous similar messages [1197018.681839] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.3.7@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1197018.699213] LustreError: Skipped 1282 previous similar messages [1197080.641106] LNetError: 23932:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [1197080.651457] LNetError: 23932:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.53@o2ib7 (105): c: 3, oc: 0, rc: 8 [1197086.641291] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1197086.651551] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 2 previous similar messages [1197155.358660] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.7.50@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1197155.376119] LustreError: Skipped 1633 previous similar messages [1197181.644058] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1197181.654318] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 3 previous similar messages [1197342.648692] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1197342.658954] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 8 previous similar messages [1197412.016025] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.110@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1197412.033572] LustreError: Skipped 2957 previous similar messages [1197684.658573] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1197684.668826] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 15 previous similar messages [1197924.360627] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.101@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1197924.378173] LustreError: Skipped 5999 previous similar messages [1198285.675942] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1198285.686200] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1198524.753055] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.106@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1198524.770591] LustreError: Skipped 6013 previous similar messages [1198887.693393] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1198887.703654] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1199125.640663] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.114@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1199125.658203] LustreError: Skipped 7396 previous similar messages [1199489.710923] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1199489.721179] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1199725.814513] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.115@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1199725.832057] LustreError: Skipped 7365 previous similar messages [1200091.728451] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1200091.738711] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1200326.100319] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.104@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1200326.117865] LustreError: Skipped 6014 previous similar messages [1200640.480382] LNet: Service thread pid 26473 was inactive for 200.50s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1200640.497493] Pid: 26473, comm: mdt02_062 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1200640.507837] Call Trace: [1200640.510488] [] ptlrpc_set_wait+0x480/0x790 [ptlrpc] [1200640.517281] [] ptlrpc_queue_wait+0x83/0x230 [ptlrpc] [1200640.524168] [] osp_remote_sync+0xd3/0x200 [osp] [1200640.530566] [] osp_attr_get+0x463/0x730 [osp] [1200640.536798] [] osp_object_init+0x16d/0x300 [osp] [1200640.543282] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1200640.550835] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1200640.557945] [] lu_object_find+0x16/0x20 [obdclass] [1200640.564625] [] mdt_object_find+0x4b/0x170 [mdt] [1200640.571024] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1200640.577608] [] mdt_reint_rec+0x83/0x210 [mdt] [1200640.583848] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1200640.590610] [] mdt_intent_open+0x82/0x3a0 [mdt] [1200640.597007] [] mdt_intent_policy+0x435/0xd80 [mdt] [1200640.603681] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1200640.610625] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1200640.617920] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1200640.624264] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1200640.631396] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1200640.639292] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1200640.645821] [] kthread+0xd1/0xe0 [1200640.650920] [] ret_from_fork_nospec_begin+0xe/0x21 [1200640.657587] [] 0xffffffffffffffff [1200640.662793] LustreError: dumping log to /tmp/lustre-log.1595426037.26473 [1200693.745980] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1200693.756235] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1200926.726500] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.9.38@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1200926.743963] LustreError: Skipped 7405 previous similar messages [1201035.287894] Lustre: 26410:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ee598767980 x1672495527682240/t0(0) o101->edb25609-39fb-4@10.49.0.63@o2ib1:452/0 lens 960/3288 e 24 to 0 dl 1595426437 ref 2 fl Interpret:/0/0 rc 0/0 [1201041.044365] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [1201041.052839] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [1201241.585895] LNet: Service thread pid 26547 was inactive for 200.50s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1201241.603008] Pid: 26547, comm: mdt03_043 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1201241.613361] Call Trace: [1201241.616008] [] lu_object_find_at+0x847/0xa60 [obdclass] [1201241.623153] [] lu_object_find+0x16/0x20 [obdclass] [1201241.629817] [] mdt_object_find+0x4b/0x170 [mdt] [1201241.636212] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1201241.642783] [] mdt_reint_rec+0x83/0x210 [mdt] [1201241.649023] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1201241.655775] [] mdt_intent_open+0x82/0x3a0 [mdt] [1201241.662171] [] mdt_intent_policy+0x435/0xd80 [mdt] [1201241.668824] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1201241.675768] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1201241.683057] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1201241.689401] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1201241.696517] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1201241.704412] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1201241.710927] [] kthread+0xd1/0xe0 [1201241.716018] [] ret_from_fork_nospec_begin+0xe/0x21 [1201241.722664] [] 0xffffffffffffffff [1201241.727868] LustreError: dumping log to /tmp/lustre-log.1595426638.26547 [1201296.763529] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1201296.773786] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1201527.386324] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.49.7.19@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1201527.403781] LustreError: Skipped 7364 previous similar messages [1201636.557424] Lustre: 24079:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8efac5b0a880 x1672495537840512/t0(0) o101->edb25609-39fb-4@10.49.0.63@o2ib1:298/0 lens 984/3288 e 24 to 0 dl 1595427038 ref 2 fl Interpret:/0/0 rc 0/0 [1201642.150185] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [1201642.158647] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [1201897.781080] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1201897.791337] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1202127.862936] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.7.30@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1202127.880397] LustreError: Skipped 6040 previous similar messages [1202237.466964] Lustre: 26761:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8efa717ec800 x1672495545829056/t0(0) o101->edb25609-39fb-4@10.49.0.63@o2ib1:144/0 lens 984/3288 e 1 to 0 dl 1595427639 ref 2 fl Interpret:/0/0 rc 0/0 [1202243.249665] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [1202243.258145] Lustre: fir-MDT0003: Connection restored to 23cfa7e0-6e5e-4 (at 10.49.0.63@o2ib1) [1202499.798674] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1202499.808932] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1202728.413102] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.3.40@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1202728.430563] LustreError: Skipped 7404 previous similar messages [1202846.240773] LNet: Service thread pid 26404 was inactive for 1204.02s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1202846.257970] Pid: 26404, comm: mdt03_022 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1202846.268321] Call Trace: [1202846.270961] [] lu_object_find_at+0x847/0xa60 [obdclass] [1202846.278067] [] lu_object_find+0x16/0x20 [obdclass] [1202846.284731] [] mdt_object_find+0x4b/0x170 [mdt] [1202846.291129] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1202846.297699] [] mdt_reint_rec+0x83/0x210 [mdt] [1202846.303920] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1202846.310662] [] mdt_intent_open+0x82/0x3a0 [mdt] [1202846.317079] [] mdt_intent_policy+0x435/0xd80 [mdt] [1202846.323742] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1202846.330698] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1202846.337981] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1202846.344326] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1202846.351442] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1202846.359329] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1202846.365838] [] kthread+0xd1/0xe0 [1202846.370926] [] ret_from_fork_nospec_begin+0xe/0x21 [1202846.377572] [] 0xffffffffffffffff [1202846.382765] LustreError: dumping log to /tmp/lustre-log.1595428243.26404 [1203102.816316] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1203102.826573] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1203328.471525] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.2.29@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1203328.488980] LustreError: Skipped 6603 previous similar messages [1203705.833851] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1203705.844112] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1203928.526885] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.49.22.23@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1203928.544433] LustreError: Skipped 7017 previous similar messages [1204307.851349] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1204307.861606] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1204529.711992] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.10.44@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1204529.729562] LustreError: Skipped 7186 previous similar messages [1204908.868802] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1204908.879063] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1205130.265003] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.103@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1205130.282545] LustreError: Skipped 6036 previous similar messages [1205510.885797] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1205510.896056] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1205730.329172] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1205730.346541] LustreError: Skipped 7435 previous similar messages [1205938.809862] LNet: Service thread pid 36462 was inactive for 200.28s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1205938.826979] Pid: 36462, comm: mdt03_099 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1205938.837329] Call Trace: [1205938.839978] [] ptlrpc_set_wait+0x480/0x790 [ptlrpc] [1205938.846756] [] ptlrpc_queue_wait+0x83/0x230 [ptlrpc] [1205938.853629] [] osp_remote_sync+0xd3/0x200 [osp] [1205938.860031] [] osp_attr_get+0x463/0x730 [osp] [1205938.866246] [] osp_object_init+0x16d/0x300 [osp] [1205938.872721] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1205938.880262] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1205938.887359] [] lu_object_find+0x16/0x20 [obdclass] [1205938.894025] [] mdt_object_find+0x4b/0x170 [mdt] [1205938.900422] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1205938.907020] [] mdt_reint_rec+0x83/0x210 [mdt] [1205938.913248] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1205938.920006] [] mdt_intent_open+0x82/0x3a0 [mdt] [1205938.926404] [] mdt_intent_policy+0x435/0xd80 [mdt] [1205938.933068] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1205938.940005] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1205938.947283] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1205938.953621] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1205938.960749] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1205938.968659] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1205938.975168] [] kthread+0xd1/0xe0 [1205938.980260] [] ret_from_fork_nospec_begin+0xe/0x21 [1205938.986908] [] 0xffffffffffffffff [1205938.992104] LustreError: dumping log to /tmp/lustre-log.1595431335.36462 [1205982.843115] LNet: Service thread pid 26507 was inactive for 200.38s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1205982.860229] Pid: 26507, comm: mdt02_072 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1205982.870575] Call Trace: [1205982.873215] [] ptlrpc_set_wait+0x480/0x790 [ptlrpc] [1205982.879998] [] ptlrpc_queue_wait+0x83/0x230 [ptlrpc] [1205982.886860] [] osp_remote_sync+0xd3/0x200 [osp] [1205982.893258] [] osp_attr_get+0x463/0x730 [osp] [1205982.899488] [] osp_object_init+0x16d/0x300 [osp] [1205982.905964] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1205982.913527] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1205982.920627] [] lu_object_find+0x16/0x20 [obdclass] [1205982.927308] [] mdt_object_find+0x4b/0x170 [mdt] [1205982.933706] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1205982.940289] [] mdt_reint_rec+0x83/0x210 [mdt] [1205982.946515] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1205982.953277] [] mdt_intent_open+0x82/0x3a0 [mdt] [1205982.959688] [] mdt_intent_policy+0x435/0xd80 [mdt] [1205982.966337] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1205982.973282] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1205982.980574] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1205982.986914] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1205982.994042] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1205983.001933] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1205983.008446] [] kthread+0xd1/0xe0 [1205983.013538] [] ret_from_fork_nospec_begin+0xe/0x21 [1205983.020198] [] 0xffffffffffffffff [1205983.025398] LustreError: dumping log to /tmp/lustre-log.1595431379.26507 [1206112.902852] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1206112.913115] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1206330.703747] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.15.11@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1206330.721308] LustreError: Skipped 7323 previous similar messages [1206333.857089] Lustre: 24501:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ef6b674de80 x1672336178159872/t0(0) o101->f2b7c93c-815b-4@10.50.0.61@o2ib2:465/0 lens 960/3288 e 24 to 0 dl 1595431735 ref 2 fl Interpret:/0/0 rc 0/0 [1206339.587793] Lustre: fir-MDT0003: Client f2b7c93c-815b-4 (at 10.50.0.61@o2ib2) reconnecting [1206339.596280] Lustre: fir-MDT0003: Connection restored to 0da406f1-5bcc-4 (at 10.50.0.61@o2ib2) [1206375.046255] LNet: Service thread pid 26586 was inactive for 412.40s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1206375.063374] Pid: 26586, comm: mdt02_090 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1206375.073738] Call Trace: [1206375.076384] [] lu_object_find_at+0x847/0xa60 [obdclass] [1206375.083494] [] lu_object_find+0x16/0x20 [obdclass] [1206375.090159] [] mdt_object_find+0x4b/0x170 [mdt] [1206375.096573] [] mdt_getattr_name_lock+0x848/0x1c30 [mdt] [1206375.103663] [] mdt_intent_getattr+0x2b5/0x480 [mdt] [1206375.110420] [] mdt_intent_policy+0x435/0xd80 [mdt] [1206375.117079] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1206375.124036] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1206375.131330] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1206375.137698] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1206375.144815] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1206375.152729] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1206375.159236] [] kthread+0xd1/0xe0 [1206375.164335] [] ret_from_fork_nospec_begin+0xe/0x21 [1206375.170989] [] 0xffffffffffffffff [1206375.176203] LustreError: dumping log to /tmp/lustre-log.1595431772.26586 [1206377.770349] Lustre: 26292:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ee693f20480 x1672327591241216/t0(0) o101->b0c2b6ec-d912-4@10.50.0.62@o2ib2:509/0 lens 968/3288 e 24 to 0 dl 1595431779 ref 2 fl Interpret:/0/0 rc 0/0 [1206383.519739] Lustre: fir-MDT0003: Client b0c2b6ec-d912-4 (at 10.50.0.62@o2ib2) reconnecting [1206383.528207] Lustre: fir-MDT0003: Connection restored to a7a03a78-2bc3-4 (at 10.50.0.62@o2ib2) [1206557.175459] Lustre: 26434:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8efa04241b00 x1672430358540480/t0(0) o101->11503ada-aab0-4@10.50.16.2@o2ib2:689/0 lens 584/3264 e 2 to 0 dl 1595431959 ref 2 fl Interpret:/0/0 rc 0/0 [1206563.696591] Lustre: fir-MDT0003: Client 11503ada-aab0-4 (at 10.50.16.2@o2ib2) reconnecting [1206563.705065] Lustre: fir-MDT0003: Connection restored to bb2f1c0f-bbbd-4 (at 10.50.16.2@o2ib2) [1206714.919971] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1206714.930224] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1206930.871566] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.17.32@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1206930.889112] LustreError: Skipped 6020 previous similar messages [1207089.330560] Lustre: 26625:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-150), not sending early reply req@ffff8ee93de25e80 x1672336179188160/t0(0) o101->f2b7c93c-815b-4@10.50.0.61@o2ib2:466/0 lens 960/3288 e 0 to 0 dl 1595432491 ref 2 fl Interpret:/0/0 rc 0/0 [1207095.696625] Lustre: fir-MDT0003: Client f2b7c93c-815b-4 (at 10.50.0.61@o2ib2) reconnecting [1207095.705091] Lustre: fir-MDT0003: Connection restored to 0da406f1-5bcc-4 (at 10.50.0.61@o2ib2) [1207133.291806] Lustre: 26468:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-150), not sending early reply req@ffff8efbf7bade80 x1672327592023040/t0(0) o101->b0c2b6ec-d912-4@10.50.0.62@o2ib2:510/0 lens 968/3288 e 0 to 0 dl 1595432535 ref 2 fl Interpret:/0/0 rc 0/0 [1207139.630899] Lustre: fir-MDT0003: Client b0c2b6ec-d912-4 (at 10.50.0.62@o2ib2) reconnecting [1207139.639370] Lustre: fir-MDT0003: Connection restored to a7a03a78-2bc3-4 (at 10.50.0.62@o2ib2) [1207164.793510] Lustre: fir-MDT0003: Client 11503ada-aab0-4 (at 10.50.16.2@o2ib2) reconnecting [1207296.160424] LNet: Service thread pid 30303 was inactive for 200.42s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1207296.177537] Pid: 30303, comm: mdt00_053 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1207296.187882] Call Trace: [1207296.190531] [] lu_object_find_at+0x847/0xa60 [obdclass] [1207296.197667] [] lu_object_find+0x16/0x20 [obdclass] [1207296.204339] [] mdt_object_find+0x4b/0x170 [mdt] [1207296.210749] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1207296.217323] [] mdt_reint_rec+0x83/0x210 [mdt] [1207296.223558] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1207296.230303] [] mdt_intent_open+0x82/0x3a0 [mdt] [1207296.236717] [] mdt_intent_policy+0x435/0xd80 [mdt] [1207296.243376] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1207296.250331] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1207296.257615] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1207296.263959] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1207296.271074] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1207296.278969] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1207296.285470] [] kthread+0xd1/0xe0 [1207296.290557] [] ret_from_fork_nospec_begin+0xe/0x21 [1207296.297206] [] 0xffffffffffffffff [1207296.302423] LustreError: dumping log to /tmp/lustre-log.1595432693.30303 [1207317.937079] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1207317.947335] LNet: 23932:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 29 previous similar messages [1207339.681653] LNet: Service thread pid 24502 was inactive for 200.01s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1207339.698766] Pid: 24502, comm: mdt00_006 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1207339.709112] Call Trace: [1207339.711756] [] lu_object_find_at+0x847/0xa60 [obdclass] [1207339.718883] [] lu_object_find+0x16/0x20 [obdclass] [1207339.725557] [] mdt_object_find+0x4b/0x170 [mdt] [1207339.731952] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1207339.738523] [] mdt_reint_rec+0x83/0x210 [mdt] [1207339.744762] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1207339.751505] [] mdt_intent_open+0x82/0x3a0 [mdt] [1207339.757892] [] mdt_intent_policy+0x435/0xd80 [mdt] [1207339.764539] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1207339.771495] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1207339.778779] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1207339.785130] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1207339.792247] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1207339.800149] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1207339.806667] [] kthread+0xd1/0xe0 [1207339.811758] [] ret_from_fork_nospec_begin+0xe/0x21 [1207339.818418] [] 0xffffffffffffffff [1207339.823617] LustreError: dumping log to /tmp/lustre-log.1595432736.24502 [1207503.526301] LNet: Service thread pid 26593 was inactive for 1163.87s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1207503.543500] Pid: 26593, comm: mdt02_096 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1207503.553843] Call Trace: [1207503.556484] [] lu_object_find_at+0x847/0xa60 [obdclass] [1207503.563625] [] lu_object_find+0x16/0x20 [obdclass] [1207503.570293] [] mdt_object_find+0x4b/0x170 [mdt] [1207503.576718] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1207503.583293] [] mdt_reint_rec+0x83/0x210 [mdt] [1207503.589535] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1207503.596282] [] mdt_intent_open+0x82/0x3a0 [mdt] [1207503.602678] [] mdt_intent_policy+0x435/0xd80 [mdt] [1207503.609333] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1207503.616275] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1207503.623572] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1207503.629910] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1207503.637038] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1207503.644927] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1207503.651444] [] kthread+0xd1/0xe0 [1207503.656535] [] ret_from_fork_nospec_begin+0xe/0x21 [1207503.663195] [] 0xffffffffffffffff [1207503.668393] LustreError: dumping log to /tmp/lustre-log.1595432900.26593 [1207530.938425] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.9.54@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1207530.955883] LustreError: Skipped 7419 previous similar messages [1207585.449608] LNet: Service thread pid 26436 was inactive for 1201.85s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1207585.466810] Pid: 26436, comm: mdt03_025 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1207585.477156] Call Trace: [1207585.479800] [] lu_object_find_at+0x847/0xa60 [obdclass] [1207585.486910] [] lu_object_find+0x16/0x20 [obdclass] [1207585.493574] [] mdt_object_find+0x4b/0x170 [mdt] [1207585.499989] [] mdt_reint_open+0x1b64/0x3240 [mdt] [1207585.506565] [] mdt_reint_rec+0x83/0x210 [mdt] [1207585.512790] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1207585.519531] [] mdt_intent_open+0x82/0x3a0 [mdt] [1207585.525927] [] mdt_intent_policy+0x435/0xd80 [mdt] [1207585.532583] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [1207585.539527] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [1207585.546816] [] tgt_enqueue+0x62/0x210 [ptlrpc] [1207585.553158] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1207585.560275] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1207585.568162] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1207585.574670] [] kthread+0xd1/0xe0 [1207585.579758] [] ret_from_fork_nospec_begin+0xe/0x21 [1207585.586407] [] 0xffffffffffffffff [1207585.591618] LustreError: dumping log to /tmp/lustre-log.1595432982.26436 [1207649.950827] Lustre: fir-MDT0003: Received new LWP connection from 10.0.10.53@o2ib7, removing former export from same NID [1207649.961889] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.53@o2ib7) [1207649.969304] Lustre: Skipped 1 previous similar message [1207690.408580] Lustre: 26729:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ecbf227d100 x1672336180025536/t0(0) o101->f2b7c93c-815b-4@10.50.0.61@o2ib2:312/0 lens 960/3288 e 24 to 0 dl 1595433092 ref 2 fl Interpret:/0/0 rc 0/0 [1207696.796128] Lustre: fir-MDT0003: Client f2b7c93c-815b-4 (at 10.50.0.61@o2ib2) reconnecting [1207713.862730] Lustre: 26473:0:(service.c:2165:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (600:6673s); client may timeout. req@ffff8ee598767980 x1672495527682240/t0(0) o101->edb25609-39fb-4@10.49.0.63@o2ib1:452/0 lens 960/560 e 24 to 0 dl 1595426437 ref 1 fl Complete:/0/0 rc 0/0 [1207713.862957] LNet: Service thread pid 26547 completed after 6672.59s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [1207713.906221] Lustre: 26473:0:(service.c:2165:ptlrpc_server_handle_request()) Skipped 6 previous similar messages [1208317.531843] Lustre: fir-MDT0003: Connection restored to 87f8f7ae-57c6-4 (at 10.51.0.14@o2ib3) [1208317.540552] Lustre: Skipped 2 previous similar messages [1208591.973202] LNetError: 23932:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [1208591.983552] LNetError: 23932:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.3@o2ib7 (107): c: 7, oc: 0, rc: 8 [1208711.195718] Lustre: fir-MDT0003: haven't heard from client 5ae5d08f-84ab-3465-2694-82b8bb873af0 (at 10.0.10.3@o2ib7) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfebdfc00, cur 1595434108 expire 1595433958 last 1595433881 [1208845.055651] Lustre: fir-MDT0003: Connection restored to 5ae5d08f-84ab-3465-2694-82b8bb873af0 (at 10.0.10.3@o2ib7) [1209502.774263] Lustre: fir-MDT0003: Connection restored to 642d8bed-bcb9-4 (at 10.49.20.34@o2ib1) [1211379.161369] Lustre: fir-MDT0003: Connection restored to 65131851-4da5-4 (at 10.51.0.67@o2ib3) [1211379.170075] Lustre: Skipped 24 previous similar messages [1211494.820066] Lustre: fir-MDT0003: Connection restored to e91b65d6-f9bf-4 (at 10.49.26.13@o2ib1) [1211905.480914] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1211905.489711] Lustre: Skipped 2 previous similar messages [1211939.286462] Lustre: fir-MDT0003: haven't heard from client 8990409c-c52c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed8bab2d800, cur 1595437336 expire 1595437186 last 1595437109 [1212780.772629] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [1212780.781327] Lustre: Skipped 1 previous similar message [1213671.683159] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1213714.336638] Lustre: fir-MDT0003: haven't heard from client c7d8798f-62a0-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8806eb400, cur 1595439111 expire 1595438961 last 1595438884 [1215756.398055] Lustre: fir-MDT0003: haven't heard from client 78c9ead4-6f6d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee995aa6800, cur 1595441153 expire 1595441003 last 1595440926 [1215836.398385] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1216641.924246] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1216666.421691] Lustre: fir-MDT0003: haven't heard from client 09810783-6bcd-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef53456b400, cur 1595442063 expire 1595441913 last 1595441836 [1218056.844175] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [1218231.778620] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1218275.466276] Lustre: fir-MDT0003: haven't heard from client afcaa074-979c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd9bc7800, cur 1595443672 expire 1595443522 last 1595443445 [1223200.604397] Lustre: fir-MDT0003: haven't heard from client df20ba62-679d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda0ca85800, cur 1595448597 expire 1595448447 last 1595448370 [1223285.146141] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1224481.555288] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1224516.640640] Lustre: fir-MDT0003: haven't heard from client 57f91c79-b277-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef53456f000, cur 1595449913 expire 1595449763 last 1595449686 [1225822.517404] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1225862.676655] Lustre: fir-MDT0003: haven't heard from client e1cdde7e-3c9f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebfeb75000, cur 1595451259 expire 1595451109 last 1595451032 [1227553.261775] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1227589.723906] Lustre: fir-MDT0003: haven't heard from client 5ec6425c-7f2a-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd434a800, cur 1595452986 expire 1595452836 last 1595452759 [1227939.734422] Lustre: fir-MDT0003: haven't heard from client cc9dec2d-faf5-4 (at 10.49.18.24@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebfeab4400, cur 1595453336 expire 1595453186 last 1595453109 [1229475.991478] Lustre: fir-MDT0003: Connection restored to cc9dec2d-faf5-4 (at 10.49.18.24@o2ib1) [1233052.906968] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458442/real 1595458442] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458449 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1233059.934154] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458449/real 1595458449] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458456 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1233066.961339] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458456/real 1595458456] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458463 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1233073.989525] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458463/real 1595458463] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458470 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1233081.016712] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458470/real 1595458470] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458477 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1233095.044092] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458484/real 1595458484] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458491 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1233095.071601] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1233116.081651] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1595458505/real 1595458505] req@ffff8eeacb7e6780 x1671791852329152/t0(0) o104->fir-MDT0003@10.49.26.35@o2ib1:15/16 lens 296/224 e 0 to 1 dl 1595458512 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1233116.109170] Lustre: 26591:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [1233150.875581] Lustre: fir-MDT0003: haven't heard from client 4b6af3e2-e6eb-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed685629400, cur 1595458547 expire 1595458397 last 1595458320 [1233150.895761] LustreError: 26591:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.49.26.35@o2ib1) failed to reply to blocking AST (req@ffff8eeacb7e6780 x1671791852329152 status 0 rc -5), evict it ns: mdt-fir-MDT0003_UUID lock: ffff8edaf5abde80/0xd0d8619aba3571ec lrc: 4/0,0 mode: PR/PR res: [0x280042b34:0x11c:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT flags: 0x60200400000020 nid: 10.49.26.35@o2ib1 remote: 0xaf0cdc0d9f0a1c76 expref: 752 pid: 31821 timeout: 1233258 lvb_type: 0 [1233150.938715] LustreError: 138-a: fir-MDT0003: A client on nid 10.49.26.35@o2ib1 was evicted due to a lock blocking callback time out: rc -5 [1233152.049659] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1235365.935377] Lustre: fir-MDT0003: haven't heard from client 6964ae35-61b8-4 (at 10.49.8.30@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7884c00, cur 1595460762 expire 1595460612 last 1595460535 [1235606.941814] Lustre: fir-MDT0003: haven't heard from client 579c4f8c-bb5e-4 (at 10.50.12.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbdafb1400, cur 1595461003 expire 1595460853 last 1595460776 [1237029.176423] Lustre: fir-MDT0003: Connection restored to 579c4f8c-bb5e-4 (at 10.50.12.10@o2ib2) [1237041.264358] Lustre: fir-MDT0003: Connection restored to 6964ae35-61b8-4 (at 10.49.8.30@o2ib1) [1239503.541845] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1239937.061514] Lustre: fir-MDT0003: haven't heard from client 234074de-a791-4 (at 10.51.0.2@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef7f9f4e000, cur 1595465333 expire 1595465183 last 1595465106 [1240165.067471] Lustre: fir-MDT0003: haven't heard from client fc6e8298-293d-4 (at 10.49.27.31@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeab2b67800, cur 1595465561 expire 1595465411 last 1595465334 [1240383.073707] Lustre: fir-MDT0003: haven't heard from client 602bead7-54c1-4 (at 10.51.1.2@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef34a716000, cur 1595465779 expire 1595465629 last 1595465552 [1240482.585883] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1241428.424030] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1241671.290534] Lustre: fir-MDT0003: Connection restored to 9247e298-4c74-4 (at 10.49.21.21@o2ib1) [1241720.111791] Lustre: fir-MDT0003: haven't heard from client 9560a244-eaf7-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee6dfb18c00, cur 1595467116 expire 1595466966 last 1595466889 [1244161.484072] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1247400.285968] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1247724.285452] Lustre: fir-MDT0003: haven't heard from client 8fce8a06-4802-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf469c800, cur 1595473120 expire 1595472970 last 1595472893 [1247738.195986] Lustre: fir-MDT0003: Connection restored to 1c060d97-c2ff-4 (at 10.50.9.39@o2ib2) [1251419.777471] Lustre: fir-MDT0003: Connection restored to d4643c8b-0261-4 (at 10.50.12.9@o2ib2) [1256565.723415] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1256587.538880] Lustre: fir-MDT0003: haven't heard from client bcd85408-43f7-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb6fae2c00, cur 1595481983 expire 1595481833 last 1595481756 [1261353.674310] Lustre: fir-MDT0003: haven't heard from client dd5266ba-4198-4 (at 10.50.5.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef93baf4c00, cur 1595486749 expire 1595486599 last 1595486522 [1276561.498655] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1276589.117566] Lustre: fir-MDT0003: haven't heard from client 5b22ac33-7d07-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef9bc756c00, cur 1595501984 expire 1595501834 last 1595501757 [1283645.908620] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1283689.318828] Lustre: fir-MDT0003: haven't heard from client 82f0b640-3b1b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee8fcfc2c00, cur 1595509084 expire 1595508934 last 1595508857 [1294928.608326] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1296100.821391] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1304213.788251] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1304245.903539] Lustre: fir-MDT0003: haven't heard from client 9eba74e3-135d-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec944181400, cur 1595529640 expire 1595529490 last 1595529413 [1311962.324818] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1312211.725243] Lustre: fir-MDT0003: Connection restored to c5f20d9d-3b96-4 (at 10.50.1.17@o2ib2) [1312271.124055] Lustre: fir-MDT0003: haven't heard from client d483f0ed-7fbf-4 (at 10.50.1.17@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1ee671c00, cur 1595537665 expire 1595537515 last 1595537438 [1324512.457804] Lustre: fir-MDT0003: haven't heard from client dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf78a8400, cur 1595549906 expire 1595549756 last 1595549679 [1324977.793131] Lustre: fir-MDT0003: Connection restored to dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) [1326025.873653] Lustre: fir-MDT0003: Connection restored to e7575d9f-8a74-4 (at 10.51.1.4@o2ib3) [1330675.627425] Lustre: fir-MDT0003: haven't heard from client 53a11268-3e45-4 (at 10.51.1.68@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbdfe1ec00, cur 1595556069 expire 1595555919 last 1595555842 [1330675.647487] Lustre: Skipped 1 previous similar message [1330897.910379] Lustre: fir-MDT0003: Connection restored to dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) [1330905.384366] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1331154.813936] Lustre: fir-MDT0003: Connection restored to 8f1a2604-3b95-4 (at 10.51.1.39@o2ib3) [1331156.828757] Lustre: fir-MDT0003: Connection restored to 926d35f3-bf4f-4 (at 10.51.1.10@o2ib3) [1331156.837464] Lustre: Skipped 44 previous similar messages [1331177.838293] Lustre: fir-MDT0003: Connection restored to 23e26230-af81-4 (at 10.51.1.8@o2ib3) [1331177.846909] Lustre: Skipped 11 previous similar messages [1331186.875954] Lustre: fir-MDT0003: Connection restored to 804ab780-33cb-4 (at 10.51.1.71@o2ib3) [1331186.884653] Lustre: Skipped 2 previous similar messages [1331443.217386] Lustre: fir-MDT0003: Connection restored to b5cff606-f853-4 (at 10.51.12.19@o2ib3) [1332137.859611] Lustre: fir-MDT0003: Connection restored to 84c97553-ca12-4 (at 10.51.12.17@o2ib3) [1332137.868400] Lustre: Skipped 1 previous similar message [1334822.627163] Lustre: fir-MDT0003: Connection restored to e7575d9f-8a74-4 (at 10.51.1.4@o2ib3) [1363024.566761] Lustre: fir-MDT0003: haven't heard from client f2934bde-8a2f-4 (at 10.50.9.6@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec6c9cc8800, cur 1595588417 expire 1595588267 last 1595588190 [1363024.586756] Lustre: Skipped 74 previous similar messages [1378616.009199] Lustre: fir-MDT0003: haven't heard from client 3ce1b8b9-68c8-4 (at 10.49.18.18@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe455fc00, cur 1595604008 expire 1595603858 last 1595603781 [1380323.386943] Lustre: fir-MDT0003: Connection restored to a229ccca-e5e6-4 (at 10.49.18.18@o2ib1) [1388409.286359] Lustre: fir-MDT0003: haven't heard from client 4c350c21-f88f-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaad6f4800, cur 1595613801 expire 1595613651 last 1595613574 [1388575.706887] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [1389604.945870] Lustre: fir-MDT0003: Connection restored to 94924e67-d766-4 (at 10.50.2.23@o2ib2) [1390759.901556] Lustre: fir-MDT0003: Connection restored to f1b08221-f6fa-4 (at 10.50.5.7@o2ib2) [1390946.504691] Lustre: fir-MDT0003: Connection restored to ba1330ea-ee3a-4 (at 10.50.5.12@o2ib2) [1401448.653150] Lustre: fir-MDT0003: haven't heard from client 9e3ea6ac-8f7b-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbee91b000, cur 1595626840 expire 1595626690 last 1595626613 [1401780.893950] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [1403808.249694] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1403851.713022] Lustre: fir-MDT0003: haven't heard from client f9779e91-312e-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edf3b3fe400, cur 1595629243 expire 1595629093 last 1595629016 [1404118.163110] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1404168.721388] Lustre: fir-MDT0003: haven't heard from client 68575cad-b90c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee000f3a400, cur 1595629560 expire 1595629410 last 1595629333 [1404596.024824] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1404620.733294] Lustre: fir-MDT0003: haven't heard from client dc32cbf0-56db-4 (at 10.51.1.4@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb7f2ab800, cur 1595630012 expire 1595629862 last 1595629785 [1404716.208484] Lustre: fir-MDT0003: Connection restored to dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) [1404721.222739] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1405041.944779] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1405044.744430] Lustre: fir-MDT0003: haven't heard from client a7439b86-bab6-4 (at 10.51.1.1@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef9f2addc00, cur 1595630436 expire 1595630286 last 1595630209 [1405044.764432] Lustre: Skipped 3 previous similar messages [1405121.017087] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1410828.125536] Lustre: fir-MDT0003: Connection restored to 40c9916b-0505-4 (at 10.50.1.59@o2ib2) [1412012.240296] Lustre: fir-MDT0003: Connection restored to e7575d9f-8a74-4 (at 10.51.1.4@o2ib3) [1413942.383346] Lustre: fir-MDT0003: Connection restored to 27f3df92-e312-4 (at 10.51.1.44@o2ib3) [1413942.392059] Lustre: Skipped 2 previous similar messages [1414480.541896] Lustre: fir-MDT0003: Connection restored to e0fca2ac-5557-4 (at 10.51.0.66@o2ib3) [1414480.550606] Lustre: Skipped 5 previous similar messages [1414673.002829] Lustre: fir-MDT0003: haven't heard from client 4a582c4c-d8fe-4 (at 10.51.1.8@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea81e17800, cur 1595640064 expire 1595639914 last 1595639837 [1414673.022797] Lustre: Skipped 1 previous similar message [1414922.794005] Lustre: fir-MDT0003: Connection restored to ac340994-7cfa-4 (at 10.51.0.13@o2ib3) [1415248.572979] Lustre: fir-MDT0003: Connection restored to e0fca2ac-5557-4 (at 10.51.0.66@o2ib3) [1420924.539962] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1420948.181501] Lustre: fir-MDT0003: haven't heard from client 7009c070-ae95-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda82f73c00, cur 1595646339 expire 1595646189 last 1595646112 [1420948.201653] Lustre: Skipped 15 previous similar messages [1426936.215999] Lustre: 24068:0:(llog_cat.c:899:llog_cat_process_or_fork()) fir-MDD0003: catlog [0x24e:0xa:0x0] crosses index zero [1426937.281113] Lustre: 26712:0:(llog_cat.c:899:llog_cat_process_or_fork()) fir-MDD0003: catlog [0x24e:0xa:0x0] crosses index zero [1426937.292680] Lustre: 26712:0:(llog_cat.c:899:llog_cat_process_or_fork()) Skipped 2 previous similar messages [1426938.798673] Lustre: 26741:0:(llog_cat.c:899:llog_cat_process_or_fork()) fir-MDD0003: catlog [0x24e:0xa:0x0] crosses index zero [1426940.924020] Lustre: 27773:0:(llog_cat.c:899:llog_cat_process_or_fork()) fir-MDD0003: catlog [0x24e:0xa:0x0] crosses index zero [1426940.935587] Lustre: 27773:0:(llog_cat.c:899:llog_cat_process_or_fork()) Skipped 6 previous similar messages [1426944.940984] Lustre: 82108:0:(llog_cat.c:899:llog_cat_process_or_fork()) fir-MDD0003: catlog [0x24e:0xa:0x0] crosses index zero [1426944.952550] Lustre: 82108:0:(llog_cat.c:899:llog_cat_process_or_fork()) Skipped 6 previous similar messages [1483708.398519] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1483748.967112] Lustre: fir-MDT0003: haven't heard from client 585c9a82-5768-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfeb80c00, cur 1595709138 expire 1595708988 last 1595708911 [1489975.136196] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd9366400, cur 1595715364 expire 1595715214 last 1595715137 [1490320.492270] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1490547.151918] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd9360800, cur 1595715936 expire 1595715786 last 1595715709 [1490794.018897] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1491020.175005] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef9bc751c00, cur 1595716409 expire 1595716259 last 1595716182 [1491335.472514] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1491562.179256] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eee111ba800, cur 1595716951 expire 1595716801 last 1595716724 [1492454.506094] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1492681.209654] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbedf4b800, cur 1595718070 expire 1595717920 last 1595717843 [1492928.080627] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1493154.230045] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaeaec9400, cur 1595718543 expire 1595718393 last 1595718316 [1493602.099621] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1493828.240293] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8dee38000, cur 1595719217 expire 1595719067 last 1595718990 [1493849.533954] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1494135.137374] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1494361.254585] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa690000, cur 1595719750 expire 1595719600 last 1595719523 [1494361.274555] Lustre: Skipped 1 previous similar message [1494630.127480] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1494856.268009] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef93bb0e000, cur 1595720245 expire 1595720095 last 1595720018 [1495128.360367] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1495355.281381] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa720800, cur 1595720744 expire 1595720594 last 1595720517 [1495684.148626] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1495910.316787] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edfc5be7400, cur 1595721299 expire 1595721149 last 1595721072 [1496479.616076] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1496706.317633] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef61cfe0000, cur 1595722095 expire 1595721945 last 1595721868 [1496954.628306] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1497181.329313] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb7f2ad000, cur 1595722570 expire 1595722420 last 1595722343 [1497460.223235] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1497686.344267] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9eadc00, cur 1595723075 expire 1595722925 last 1595722848 [1498502.229392] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1498728.372062] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eedf9a49400, cur 1595724117 expire 1595723967 last 1595723890 [1499336.686202] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1499563.394704] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe4628c00, cur 1595724952 expire 1595724802 last 1595724725 [1500024.491274] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1500251.413410] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd6a47000, cur 1595725640 expire 1595725490 last 1595725413 [1500564.724841] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1500791.427763] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef178ffa800, cur 1595726180 expire 1595726030 last 1595725953 [1501395.420325] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1501621.450203] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8fbe10800, cur 1595727010 expire 1595726860 last 1595726783 [1501987.766271] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1502214.466485] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1f6f64400, cur 1595727603 expire 1595727453 last 1595727376 [1502820.571217] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1503047.489188] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1ee671400, cur 1595728436 expire 1595728286 last 1595728209 [1503331.799767] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1503558.502248] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf95ff000, cur 1595728947 expire 1595728797 last 1595728720 [1503839.817499] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1504066.517105] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef3e5edcc00, cur 1595729455 expire 1595729305 last 1595729228 [1504312.386396] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1504538.530965] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbf9c49800, cur 1595729927 expire 1595729777 last 1595729700 [1504842.842310] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1505069.546331] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7848800, cur 1595730458 expire 1595730308 last 1595730231 [1505353.858028] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1505580.559137] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebb39e3800, cur 1595730969 expire 1595730819 last 1595730742 [1506084.448910] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1506310.579387] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef7f9f4d000, cur 1595731699 expire 1595731549 last 1595731472 [1507774.925458] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1508001.631091] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef178ff8000, cur 1595733390 expire 1595733240 last 1595733163 [1509136.523785] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1509362.665235] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed8bab2cc00, cur 1595734751 expire 1595734601 last 1595734524 [1509938.554799] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1510164.692261] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf4f7c800, cur 1595735553 expire 1595735403 last 1595735326 [1511088.584340] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1511314.724291] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90bf91800, cur 1595736703 expire 1595736553 last 1595736476 [1512677.769459] Lustre: fir-MDT0003: haven't heard from client b9a8bdb8-2050-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee262675400, cur 1595738066 expire 1595737916 last 1595737839 [1512686.063708] Lustre: fir-MDT0003: Connection restored to 462cf204-e318-4 (at 10.49.7.3@o2ib1) [1512912.767656] Lustre: fir-MDT0003: haven't heard from client 462cf204-e318-4 (at 10.49.7.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed984f70800, cur 1595738301 expire 1595738151 last 1595738074 [1516589.777201] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [1516815.881468] Lustre: fir-MDT0003: haven't heard from client 2726b051-a51b-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edad4a53c00, cur 1595742204 expire 1595742054 last 1595741977 [1517610.595211] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [1518089.918956] Lustre: fir-MDT0003: haven't heard from client 5c47c1d8-5bb2-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90b557800, cur 1595743478 expire 1595743328 last 1595743251 [1522612.469852] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [1523041.064885] Lustre: fir-MDT0003: haven't heard from client a8fb1ad0-42f0-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee9c1ab4800, cur 1595748429 expire 1595748279 last 1595748202 [1523226.433916] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [1523832.088155] Lustre: fir-MDT0003: haven't heard from client cb229453-9271-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee94f298800, cur 1595749220 expire 1595749070 last 1595748993 [1525178.177358] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [1525908.149716] Lustre: fir-MDT0003: haven't heard from client de752f21-179f-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef934b48c00, cur 1595751296 expire 1595751146 last 1595751069 [1528525.036264] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1528549.227023] Lustre: fir-MDT0003: haven't heard from client a495aacc-9f4c-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda8d7b0c00, cur 1595753937 expire 1595753787 last 1595753710 [1562601.939361] Lustre: fir-MDT0003: Connection restored to 9d147bcd-62a0-4 (at 10.49.25.17@o2ib1) [1565292.279595] Lustre: fir-MDT0003: haven't heard from client c40f5e2f-8427-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee0dea01000, cur 1595790679 expire 1595790529 last 1595790452 [1565331.367363] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1565936.329765] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1565986.299312] Lustre: fir-MDT0003: haven't heard from client 31118bf2-25fa-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd4349c00, cur 1595791373 expire 1595791223 last 1595791146 [1566651.306439] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1566691.319626] Lustre: fir-MDT0003: haven't heard from client 53a2fc2a-a7a8-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eda8d7b5000, cur 1595792078 expire 1595791928 last 1595791851 [1567180.261630] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1567255.337014] Lustre: fir-MDT0003: haven't heard from client 0ff73ab1-da3a-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed96234e800, cur 1595792642 expire 1595792492 last 1595792415 [1568168.313091] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1568211.358441] Lustre: fir-MDT0003: haven't heard from client 8879392d-d161-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb9b100c00, cur 1595793598 expire 1595793448 last 1595793371 [1569287.583634] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1569324.389957] Lustre: fir-MDT0003: haven't heard from client 0689878c-1c98-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef6b2927800, cur 1595794711 expire 1595794561 last 1595794484 [1570617.699378] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1570619.429864] Lustre: fir-MDT0003: haven't heard from client a6ea2381-281d-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1b2e5cc00, cur 1595796006 expire 1595795856 last 1595795779 [1571966.860376] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1572024.464579] Lustre: fir-MDT0003: haven't heard from client 50594646-64ee-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed730a65000, cur 1595797411 expire 1595797261 last 1595797184 [1573748.282940] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1573807.513775] Lustre: fir-MDT0003: haven't heard from client 60f83ed3-455a-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efa9e7a8000, cur 1595799194 expire 1595799044 last 1595798967 [1574024.520739] Lustre: fir-MDT0003: haven't heard from client 160d473f-2417-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef6b2926000, cur 1595799411 expire 1595799261 last 1595799184 [1574080.358036] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1574352.063137] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1574408.528766] Lustre: fir-MDT0003: haven't heard from client 29dfe9a9-e79e-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd434f400, cur 1595799795 expire 1595799645 last 1595799568 [1575568.737484] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1575583.562462] Lustre: fir-MDT0003: haven't heard from client 5a8b020a-de19-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb3aa9d800, cur 1595800970 expire 1595800820 last 1595800743 [1588727.678235] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1588742.916128] Lustre: fir-MDT0003: haven't heard from client 4d3024be-1174-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed7fd704400, cur 1595814129 expire 1595813979 last 1595813902 [1588975.458927] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1589030.926498] Lustre: fir-MDT0003: haven't heard from client 359f5834-e987-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed730a62800, cur 1595814417 expire 1595814267 last 1595814190 [1589697.973794] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1589705.948232] Lustre: fir-MDT0003: haven't heard from client e302b834-0803-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb3aa99400, cur 1595815092 expire 1595814942 last 1595814865 [1597212.743755] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1597274.161997] Lustre: fir-MDT0003: haven't heard from client 7dcae10a-71c9-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ede73e69000, cur 1595822660 expire 1595822510 last 1595822433 [1597952.178000] Lustre: fir-MDT0003: haven't heard from client 8211c5df-2445-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef934b4b000, cur 1595823338 expire 1595823188 last 1595823111 [1598025.104164] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1598173.668119] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1598251.186769] Lustre: fir-MDT0003: haven't heard from client f9ab1ab9-6433-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee9c1ab7c00, cur 1595823637 expire 1595823487 last 1595823410 [1599223.369775] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1599282.216860] Lustre: fir-MDT0003: haven't heard from client 0c0e316c-4a6d-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaa16b9400, cur 1595824668 expire 1595824518 last 1595824441 [1599652.662854] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1599713.229310] Lustre: fir-MDT0003: haven't heard from client feb42cb9-1be0-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed63e23e400, cur 1595825099 expire 1595824949 last 1595824872 [1600493.626783] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1600554.254782] Lustre: fir-MDT0003: haven't heard from client e5c21263-f7c8-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf7882000, cur 1595825940 expire 1595825790 last 1595825713 [1601077.290021] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1601098.269582] Lustre: fir-MDT0003: haven't heard from client 051ac53c-d09d-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8fbe10800, cur 1595826484 expire 1595826334 last 1595826257 [1601533.961224] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1601581.283896] Lustre: fir-MDT0003: haven't heard from client 2be31958-0521-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef3e5ed9000, cur 1595826967 expire 1595826817 last 1595826740 [1602063.297919] Lustre: fir-MDT0003: haven't heard from client 09e66f9d-74c2-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee8f8351000, cur 1595827449 expire 1595827299 last 1595827222 [1602108.965539] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1602261.331108] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1602335.306768] Lustre: fir-MDT0003: haven't heard from client ea9a989c-6a21-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee94f299800, cur 1595827721 expire 1595827571 last 1595827494 [1602703.030044] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1602740.317987] Lustre: fir-MDT0003: haven't heard from client b6f10625-84bd-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef9f2ada000, cur 1595828126 expire 1595827976 last 1595827899 [1602894.006640] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1602956.324232] Lustre: fir-MDT0003: haven't heard from client 7c3a0bf4-0442-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed81850e000, cur 1595828342 expire 1595828192 last 1595828115 [1603530.593430] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1603574.342572] Lustre: fir-MDT0003: haven't heard from client c813bf91-178a-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed96234d000, cur 1595828960 expire 1595828810 last 1595828733 [1603979.738113] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1604035.356421] Lustre: fir-MDT0003: haven't heard from client 01e97feb-8f40-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef178ff8c00, cur 1595829421 expire 1595829271 last 1595829194 [1604760.377300] Lustre: fir-MDT0003: haven't heard from client 466015f6-222f-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee9c1ab1c00, cur 1595830146 expire 1595829996 last 1595829919 [1604921.068746] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1605549.399757] Lustre: fir-MDT0003: haven't heard from client 87f8f7ae-57c6-4 (at 10.51.0.14@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef934b4e000, cur 1595830935 expire 1595830785 last 1595830708 [1605587.671618] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1605760.196423] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1607751.247804] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1607794.463241] Lustre: fir-MDT0003: haven't heard from client b3e84db6-d082-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef9f2adcc00, cur 1595833180 expire 1595833030 last 1595832953 [1607794.483384] Lustre: Skipped 2 previous similar messages [1608305.791040] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1608355.481302] Lustre: fir-MDT0003: haven't heard from client aba3b188-7955-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed5eba67800, cur 1595833741 expire 1595833591 last 1595833514 [1608885.495745] Lustre: fir-MDT0003: haven't heard from client a5bad484-e459-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee8d6b44400, cur 1595834271 expire 1595834121 last 1595834044 [1609173.900632] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1609332.329084] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1609400.510344] Lustre: fir-MDT0003: haven't heard from client 8794f36e-f0a6-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed81ee71400, cur 1595834786 expire 1595834636 last 1595834559 [1609819.455255] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1610319.307851] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1610373.536969] Lustre: fir-MDT0003: haven't heard from client 6ccd8184-f49b-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edad8357c00, cur 1595835759 expire 1595835609 last 1595835532 [1610373.557117] Lustre: Skipped 1 previous similar message [1610950.475772] Lustre: fir-MDT0003: Connection restored to 1e6defdf-6f9b-4 (at 10.49.23.14@o2ib1) [1610974.558289] Lustre: fir-MDT0003: haven't heard from client f23a08bb-cb02-4 (at 10.49.23.14@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1b77c7400, cur 1595836360 expire 1595836210 last 1595836133 [1631686.148373] Lustre: fir-MDT0003: haven't heard from client 42930117-58c1-4 (at 10.50.14.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef5272eec00, cur 1595857071 expire 1595856921 last 1595856844 [1639264.594217] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1639266.003415] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1639268.242892] Lustre: fir-MDT0003: Connection restored to 926d35f3-bf4f-4 (at 10.51.1.10@o2ib3) [1639268.251591] Lustre: Skipped 1 previous similar message [1639304.589190] Lustre: fir-MDT0003: Connection restored to 59b05271-2952-4 (at 10.51.1.5@o2ib3) [1639304.597811] Lustre: Skipped 4 previous similar messages [1639309.490269] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1639309.498885] Lustre: Skipped 2 previous similar messages [1640003.588098] Lustre: fir-MDT0003: Connection restored to e0fca2ac-5557-4 (at 10.51.0.66@o2ib3) [1646694.709541] Lustre: fir-MDT0003: Connection restored to 671be619-543e-4 (at 10.51.0.11@o2ib3) [1646699.983042] Lustre: fir-MDT0003: Connection restored to 87f8f7ae-57c6-4 (at 10.51.0.14@o2ib3) [1647191.719365] Lustre: fir-MDT0003: Connection restored to 28e5f2ac-c987-4 (at 10.51.1.9@o2ib3) [1647233.678155] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1647913.477433] Lustre: fir-MDT0003: Connection restored to e7575d9f-8a74-4 (at 10.51.1.4@o2ib3) [1651311.173320] Lustre: fir-MDT0003: Connection restored to 65131851-4da5-4 (at 10.51.0.67@o2ib3) [1651389.712258] Lustre: fir-MDT0003: haven't heard from client 910b4c30-1815-4 (at 10.51.1.69@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea81e17000, cur 1595876774 expire 1595876624 last 1595876547 [1651463.995515] Lustre: fir-MDT0003: Connection restored to e3c46367-7608-4 (at 10.51.1.69@o2ib3) [1652630.747359] Lustre: fir-MDT0003: haven't heard from client 60c26422-f08f-4 (at 10.51.1.3@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef53456f800, cur 1595878015 expire 1595877865 last 1595877788 [1652718.123739] Lustre: fir-MDT0003: Connection restored to dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) [1654588.659812] LustreError: 26289:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [1655818.835577] Lustre: fir-MDT0003: haven't heard from client a10ad184-848d-4 (at 10.51.1.14@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee94f298000, cur 1595881203 expire 1595881053 last 1595880976 [1655818.855635] Lustre: Skipped 1 previous similar message [1655913.819936] Lustre: fir-MDT0003: Connection restored to 214cead4-00a3-4 (at 10.51.1.11@o2ib3) [1656250.025700] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1656259.341049] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1656262.333615] Lustre: fir-MDT0003: Connection restored to 72b1a979-22f6-4 (at 10.51.1.18@o2ib3) [1656262.342321] Lustre: Skipped 1 previous similar message [1656269.918878] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1657309.877234] Lustre: fir-MDT0003: haven't heard from client 671be619-543e-4 (at 10.51.0.11@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edac0e6c400, cur 1595882694 expire 1595882544 last 1595882467 [1657309.897318] Lustre: Skipped 5 previous similar messages [1657323.732959] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1657323.750360] LustreError: Skipped 1562 previous similar messages [1657385.878900] Lustre: fir-MDT0003: haven't heard from client 90ddfc2d-dd13-4 (at 10.51.1.16@o2ib3) in 223 seconds. I think it's dead, and I am evicting it. exp ffff8ef61cfe2c00, cur 1595882770 expire 1595882620 last 1595882547 [1657385.898958] Lustre: Skipped 4 previous similar messages [1657445.198131] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1657454.302689] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1657457.371201] Lustre: fir-MDT0003: Connection restored to 72b1a979-22f6-4 (at 10.51.1.18@o2ib3) [1657457.379907] Lustre: Skipped 1 previous similar message [1657465.049662] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1657506.265984] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1657815.516868] Lustre: fir-MDT0003: Connection restored to 072b4985-6514-4 (at 10.51.1.17@o2ib3) [1657815.525599] Lustre: Skipped 1 previous similar message [1658725.884679] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1658744.186895] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1659171.926180] Lustre: fir-MDT0003: haven't heard from client af651c57-988f-4 (at 10.51.1.2@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eed6ddb0c00, cur 1595884556 expire 1595884406 last 1595884329 [1659171.946148] Lustre: Skipped 2 previous similar messages [1659270.320200] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1659410.402829] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1660275.680928] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1660275.689549] Lustre: Skipped 1 previous similar message [1660338.294898] Lustre: fir-MDT0003: Connection restored to e7575d9f-8a74-4 (at 10.51.1.4@o2ib3) [1660400.960423] Lustre: fir-MDT0003: haven't heard from client be24d028-037d-4 (at 10.51.0.14@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecbe7e4a800, cur 1595885785 expire 1595885635 last 1595885558 [1663041.450051] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1663046.152665] Lustre: fir-MDT0003: Connection restored to dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) [1663122.252893] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1664469.286968] Lustre: fir-MDT0003: Connection restored to 383558b6-8b48-4 (at 10.51.1.12@o2ib3) [1664469.295667] Lustre: Skipped 1 previous similar message [1664889.089670] Lustre: fir-MDT0003: haven't heard from client ad4f9e73-27fe-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee9da224000, cur 1595890273 expire 1595890123 last 1595890046 [1665197.288699] Lustre: fir-MDT0003: Connection restored to 4b806782-490c-4 (at 10.50.16.10@o2ib2) [1666981.679763] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1667030.003348] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1672824.294011] Lustre: fir-MDT0003: haven't heard from client 6faeabc9-edf7-4 (at 10.51.1.5@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec8a972a800, cur 1595898208 expire 1595898058 last 1595897981 [1672886.618522] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1672895.859023] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1672899.732167] Lustre: fir-MDT0003: Connection restored to 72b1a979-22f6-4 (at 10.51.1.18@o2ib3) [1672899.740866] Lustre: Skipped 1 previous similar message [1672905.846587] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1672914.524637] Lustre: fir-MDT0003: Connection restored to dc0e81e7-68f0-4 (at 10.51.1.3@o2ib3) [1672914.533251] Lustre: Skipped 47 previous similar messages [1673796.319662] Lustre: fir-MDT0003: haven't heard from client 7c478722-6522-4 (at 10.51.1.2@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbe542b000, cur 1595899180 expire 1595899030 last 1595898953 [1673796.339646] Lustre: Skipped 71 previous similar messages [1673855.448053] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1673855.456768] Lustre: Skipped 1 previous similar message [1673864.655754] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1673869.090676] Lustre: fir-MDT0003: Connection restored to 72b1a979-22f6-4 (at 10.51.1.18@o2ib3) [1673869.099381] Lustre: Skipped 1 previous similar message [1673874.646875] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1673884.629430] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1673884.638071] Lustre: Skipped 47 previous similar messages [1674101.328946] Lustre: fir-MDT0003: haven't heard from client 7e9a2c08-d11f-4 (at 10.51.1.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec5a0273400, cur 1595899485 expire 1595899335 last 1595899258 [1674101.349003] Lustre: Skipped 53 previous similar messages [1674164.274292] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1674164.282991] Lustre: Skipped 1 previous similar message [1689202.754720] Lustre: fir-MDT0003: haven't heard from client 52329457-c30b-4 (at 10.51.1.14@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1772ccc00, cur 1595914586 expire 1595914436 last 1595914359 [1689202.774778] Lustre: Skipped 53 previous similar messages [1689263.835919] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1689263.844620] Lustre: Skipped 53 previous similar messages [1689273.090075] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1689283.062846] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1689283.071556] Lustre: Skipped 2 previous similar messages [1690725.653685] Lustre: fir-MDT0003: Connection restored to e443e755-cf83-4 (at 10.49.31.8@o2ib1) [1690725.662387] Lustre: Skipped 49 previous similar messages [1691375.983460] Lustre: fir-MDT0003: Client d0bda935-80fd-4 (at 10.50.17.6@o2ib2) reconnecting [1691375.983540] Lustre: fir-MDT0003: Connection restored to dd5cd45e-bf1a-4 (at 10.50.17.12@o2ib2) [1691376.000709] Lustre: Skipped 1 previous similar message [1722881.754962] Lustre: fir-MDT0003: Connection restored to 671be619-543e-4 (at 10.51.0.11@o2ib3) [1722881.763663] Lustre: Skipped 1 previous similar message [1722893.442024] Lustre: fir-MDT0003: Connection restored to 87f8f7ae-57c6-4 (at 10.51.0.14@o2ib3) [1723895.762478] Lustre: fir-MDT0003: haven't heard from client 79a8d3f0-02e0-4 (at 10.51.1.45@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea35b95c00, cur 1595949278 expire 1595949128 last 1595949051 [1723895.782530] Lustre: Skipped 53 previous similar messages [1723956.009010] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1723966.063616] Lustre: fir-MDT0003: Connection restored to 79b123b0-cacb-4 (at 10.51.1.14@o2ib3) [1723969.318392] Lustre: fir-MDT0003: Connection restored to 72b1a979-22f6-4 (at 10.51.1.18@o2ib3) [1723969.327094] Lustre: Skipped 1 previous similar message [1723976.154532] Lustre: fir-MDT0003: Connection restored to b0573b92-2587-4 (at 10.51.1.13@o2ib3) [1723985.294827] Lustre: fir-MDT0003: Connection restored to 91ac32ec-97fe-4 (at 10.51.1.2@o2ib3) [1723985.303457] Lustre: Skipped 39 previous similar messages [1727078.846731] Lustre: fir-MDT0003: haven't heard from client 2705e877-b148-4 (at 10.51.1.16@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebba9e1000, cur 1595952461 expire 1595952311 last 1595952234 [1727078.866794] Lustre: Skipped 53 previous similar messages [1727450.455966] Lustre: fir-MDT0003: Connection restored to 32b3c36e-ea5f-4 (at 10.50.7.38@o2ib2) [1728469.473174] Lustre: fir-MDT0003: Connection restored to 7f7804ce-f86e-4 (at 10.51.1.32@o2ib3) [1728472.182955] Lustre: fir-MDT0003: Connection restored to 58e7343e-18ea-4 (at 10.51.1.24@o2ib3) [1728476.244453] Lustre: fir-MDT0003: Connection restored to 11eab6d5-308c-4 (at 10.51.1.40@o2ib3) [1728479.218055] Lustre: fir-MDT0003: Connection restored to 295f4622-e43c-4 (at 10.51.1.34@o2ib3) [1728488.635823] Lustre: fir-MDT0003: Connection restored to f8f98ffc-b298-4 (at 10.51.1.21@o2ib3) [1728497.519839] Lustre: fir-MDT0003: Connection restored to 5aa377c7-37cd-4 (at 10.51.1.66@o2ib3) [1728497.528542] Lustre: Skipped 6 previous similar messages [1728519.471715] Lustre: fir-MDT0003: Connection restored to e57d0e2a-3de9-4 (at 10.51.1.22@o2ib3) [1728519.480412] Lustre: Skipped 14 previous similar messages [1728652.112900] Lustre: fir-MDT0003: Connection restored to 8da5de21-0bbb-4 (at 10.51.1.16@o2ib3) [1728652.121615] Lustre: Skipped 3 previous similar messages [1728855.226379] Lustre: fir-MDT0003: Connection restored to 9cae4620-053e-4 (at 10.51.1.15@o2ib3) [1728855.235084] Lustre: Skipped 11 previous similar messages [1730516.404551] Lustre: fir-MDT0003: Connection restored to e7575d9f-8a74-4 (at 10.51.1.4@o2ib3) [1730516.413160] Lustre: Skipped 10 previous similar messages [1730694.843788] Lustre: fir-MDT0003: Connection restored to 6ac2af02-a753-4 (at 10.51.1.45@o2ib3) [1732010.826932] Lustre: fir-MDT0003: Connection restored to 3bbb827c-88c0-4 (at 10.49.26.35@o2ib1) [1732046.987189] Lustre: fir-MDT0003: haven't heard from client b13a17b5-5d5b-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef6b2927400, cur 1595957429 expire 1595957279 last 1595957202 [1732047.007343] Lustre: Skipped 44 previous similar messages [1732764.851051] Lustre: fir-MDT0003: Client edccf0f6-263c-4 (at 10.50.17.2@o2ib2) reconnecting [1732764.851414] Lustre: fir-MDT0003: Connection restored to 0591bcf6-b88d-4 (at 10.50.17.3@o2ib2) [1732764.868197] Lustre: Skipped 1 previous similar message [1732801.804986] Lustre: fir-MDT0003: Client 8e3fa4c3-88a1-4 (at 10.50.17.10@o2ib2) reconnecting [1732801.813536] Lustre: fir-MDT0003: Connection restored to 0ddf7418-68c1-4 (at 10.50.17.10@o2ib2) [1732801.822339] Lustre: Skipped 1 previous similar message [1733368.853574] LustreError: 11-0: fir-MDT0001-osp-MDT0003: operation mds_statfs to node 10.0.10.52@o2ib7 failed: rc = -107 [1733368.864527] LustreError: Skipped 91 previous similar messages [1733368.870459] Lustre: fir-MDT0001-osp-MDT0003: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1733411.897791] LustreError: 137-5: fir-MDT0001_UUID: not available for connect from 10.49.0.71@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1733411.915246] LustreError: Skipped 1 previous similar message [1733430.661295] LustreError: 137-5: fir-MDT0001_UUID: not available for connect from 10.49.23.25@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1733430.678846] LustreError: Skipped 785 previous similar messages [1733468.369509] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.52@o2ib7) [1733517.397603] LustreError: 137-5: fir-MDT0001_UUID: not available for connect from 10.0.10.116@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1733517.415142] LustreError: Skipped 657 previous similar messages [1733568.240047] Lustre: fir-MDT0001-osp-MDT0003: Connection restored to 10.0.10.52@o2ib7 (at 10.0.10.52@o2ib7) [1735916.403677] Lustre: fir-MDT0003: Connection restored to 6b6513bc-e39e-4 (at 10.51.1.1@o2ib3) [1735921.872582] Lustre: fir-MDT0003: Connection restored to 8034fdf1-4d4a-4 (at 10.51.1.57@o2ib3) [1735921.881285] Lustre: Skipped 1 previous similar message [1735925.376473] Lustre: fir-MDT0003: Connection restored to d1e14ee7-9fca-4 (at 10.51.1.33@o2ib3) [1735931.256055] Lustre: fir-MDT0003: Connection restored to 2d819734-cc5e-4 (at 10.51.1.65@o2ib3) [1735935.405982] Lustre: fir-MDT0003: Connection restored to 383558b6-8b48-4 (at 10.51.1.12@o2ib3) [1735952.144779] Lustre: fir-MDT0003: Connection restored to 28e5f2ac-c987-4 (at 10.51.1.9@o2ib3) [1735952.153390] Lustre: Skipped 2 previous similar messages [1735976.196071] Lustre: fir-MDT0003: Connection restored to 14e69a17-487b-4 (at 10.51.1.28@o2ib3) [1735976.204773] Lustre: Skipped 3 previous similar messages [1739012.858751] Lustre: fir-MDT0003: Connection restored to dde7e138-7454-4 (at 10.51.1.60@o2ib3) [1739012.867452] Lustre: Skipped 3 previous similar messages [1743633.712931] Lustre: fir-MDT0003: Connection restored to a5d9d05b-b770-4 (at 10.49.27.31@o2ib1) [1743658.951827] Lustre: fir-MDT0003: Connection restored to 4d96a30d-08fa-4 (at 10.50.9.6@o2ib2) [1744007.324018] Lustre: fir-MDT0003: haven't heard from client 682af5e3-4ab9-4 (at 10.51.1.60@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed81ee77c00, cur 1595969389 expire 1595969239 last 1595969162 [1746333.337840] Lustre: Failing over fir-MDT0003 [1746333.369935] Lustre: fir-MDT0003: Not available for connect from 10.50.16.1@o2ib2 (stopping) [1746333.378460] Lustre: Skipped 2 previous similar messages [1746333.616817] LustreError: 65582:0:(osp_object.c:594:osp_attr_get()) fir-MDT0002-osp-MDT0003:osp_attr_get update error [0x20000000a:0x2:0x0]: rc = -108 [1746333.642966] LustreError: 65582:0:(llog_cat.c:449:llog_cat_close()) fir-MDT0002-osp-MDT0003: failure destroying log during cleanup: rc = -108 [1746333.876464] Lustre: fir-MDT0003: Not available for connect from 10.50.4.42@o2ib2 (stopping) [1746333.884991] Lustre: Skipped 98 previous similar messages [1746334.007949] LustreError: 26813:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.49.17.23@o2ib1 arrived at 1595971715 with bad export cookie 15048885372511489572 [1746334.023773] LustreError: 26813:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) ### ### ns: mdt-fir-MDT0003_UUID lock: ffff8eeb48e50240/0xd0d861ab54367102 lrc: 3/0,0 mode: PR/PR res: [0x280041fec:0x4c95:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT flags: 0x40200000000000 nid: 10.49.17.23@o2ib1 remote: 0x601d88436dd07e7d expref: 4 pid: 26596 timeout: 0 lvb_type: 0 [1746334.055273] LustreError: 26813:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) Skipped 1 previous similar message [1746334.548210] LustreError: 24054:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.50.17.14@o2ib2 arrived at 1595971716 with bad export cookie 15048885373794854452 [1746334.548318] LustreError: 26813:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) ### ### ns: mdt-fir-MDT0003_UUID lock: ffff8ee94d6b98c0/0xd0d861ab542e4bbf lrc: 3/0,0 mode: PR/PR res: [0x280042b32:0x8e2d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT flags: 0x40200000000000 nid: 10.50.17.14@o2ib2 remote: 0x7751643c3dfc2daa expref: 145569 pid: 26596 timeout: 0 lvb_type: 0 [1746334.548321] LustreError: 26813:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) Skipped 1 previous similar message [1746334.606118] LustreError: 24054:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) Skipped 4 previous similar messages [1746334.887534] Lustre: fir-MDT0003: Not available for connect from 10.50.7.29@o2ib2 (stopping) [1746334.896063] Lustre: Skipped 155 previous similar messages [1746335.992963] LustreError: 26727:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.50.1.55@o2ib2 arrived at 1595971717 with bad export cookie 15048885374141697767 [1746336.008688] LustreError: 26727:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) Skipped 1 previous similar message [1746336.018793] LustreError: 26727:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) ### ### ns: mdt-fir-MDT0003_UUID lock: ffff8ec96104af40/0xd0d861ab543ab4ae lrc: 3/0,0 mode: CR/CR res: [0x280043185:0xd32:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x40000000000000 nid: 10.50.1.55@o2ib2 remote: 0x7363c1e60a7b3d80 expref: 32 pid: 26719 timeout: 0 lvb_type: 0 [1746336.050133] LustreError: 26727:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) Skipped 4 previous similar messages [1746336.930338] Lustre: fir-MDT0003: Not available for connect from 10.49.8.21@o2ib1 (stopping) [1746336.938864] Lustre: Skipped 120 previous similar messages [1746338.293849] LustreError: 24054:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.49.26.36@o2ib1 arrived at 1595971719 with bad export cookie 15048885382073858304 [1746338.309663] LustreError: 24054:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) Skipped 2 previous similar messages [1746338.319855] LustreError: 24054:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) ### ### ns: mdt-fir-MDT0003_UUID lock: ffff8edbc5ecd7c0/0xd0d861ab544ff33e lrc: 3/0,0 mode: PR/PR res: [0x2800428eb:0x2b2d:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT flags: 0x40200000000000 nid: 10.49.26.36@o2ib1 remote: 0xdc3f76499ac732ba expref: 265 pid: 26693 timeout: 0 lvb_type: 0 [1746338.351529] LustreError: 24054:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) Skipped 1 previous similar message [1746341.024607] Lustre: fir-MDT0003: Not available for connect from 10.49.22.7@o2ib1 (stopping) [1746341.033142] Lustre: Skipped 153 previous similar messages [1746342.471647] LustreError: 26384:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.49.0.63@o2ib1 arrived at 1595971724 with bad export cookie 15048885384512427325 [1746342.487376] LustreError: 26384:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) Skipped 5 previous similar messages [1746343.170480] LustreError: 26384:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) ### ### ns: mdt-fir-MDT0003_UUID lock: ffff8eea8839fbc0/0xd0d861ab5443451a lrc: 3/0,0 mode: PR/PR res: [0x280041e73:0xf2b0:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT flags: 0x40200000000000 nid: 10.50.1.66@o2ib2 remote: 0xd5e00b16498638a6 expref: 47 pid: 26580 timeout: 0 lvb_type: 0 [1746343.201989] LustreError: 26384:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) Skipped 4 previous similar messages [1746349.049809] Lustre: fir-MDT0003: Not available for connect from 10.50.3.1@o2ib2 (stopping) [1746349.058253] Lustre: Skipped 377 previous similar messages [1746353.084637] LustreError: 26727:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.50.9.72@o2ib2 arrived at 1595971734 with bad export cookie 15048885360422267210 [1746353.100360] LustreError: 26727:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) Skipped 3 previous similar messages [1746353.110555] LustreError: 26727:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) ### ### ns: mdt-fir-MDT0003_UUID lock: ffff8ef5e98bc380/0xd0d861ab621957a4 lrc: 3/0,0 mode: CR/CR res: [0x28004203d:0x3dd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x40000000000000 nid: 10.50.9.72@o2ib2 remote: 0x2dc33174f388b524 expref: 2 pid: 26599 timeout: 0 lvb_type: 0 [1746353.141797] LustreError: 26727:0:(ldlm_lock.c:2713:ldlm_lock_dump_handle()) Skipped 2 previous similar messages [1746366.560985] Lustre: fir-MDT0003: Not available for connect from 10.50.9.58@o2ib2 (stopping) [1746366.569515] Lustre: Skipped 450 previous similar messages [1746370.130674] LustreError: 65703:0:(qsd_reint.c:56:qsd_reint_completion()) fir-MDT0003: failed to enqueue global quota lock, glb fid:[0x200000006:0x20000:0x0], rc:-108 [1746370.145622] LustreError: 65703:0:(qsd_reint.c:56:qsd_reint_completion()) Skipped 2 previous similar messages [1746376.183367] LustreError: 65706:0:(qsd_reint.c:56:qsd_reint_completion()) fir-MDT0003: failed to enqueue global quota lock, glb fid:[0x200000006:0x10000:0x0], rc:-108 [1746376.198330] LustreError: 65706:0:(qsd_reint.c:56:qsd_reint_completion()) Skipped 2 previous similar messages [1746379.978452] Lustre: server umount fir-MDT0003 complete [1746415.412218] LNet: Removed LNI 10.0.10.54@o2ib7 [1746421.389046] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1746421.396847] alg: No test for adler32 (adler32-zlib) [1746422.226540] Lustre: Lustre: Build Version: 2.12.5_2_g8ac362a [1746422.354204] LNet: 65987:0:(config.c:1642:lnet_inet_enumerate()) lnet: Ignoring interface em2: it's down [1746422.364195] LNet: Using FastReg for registration [1746422.380361] LNet: Added LNI 10.0.10.54@o2ib7 [8/256/0/180] [1746423.680537] LDISKFS-fs (dm-0): file extents enabled, maximum tree depth=5 [1746425.445778] LDISKFS-fs (dm-0): mounted filesystem with ordered data mode. Opts: user_xattr,errors=remount-ro,acl,no_mbcache,nodelalloc [1746428.353571] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1746430.353621] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 2 seconds [1746431.353651] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 3 seconds [1746431.424508] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.50.12.4@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746431.943475] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.49.25.1@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746431.960930] LustreError: Skipped 4 previous similar messages [1746433.353707] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 5 seconds [1746433.363964] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [1746433.450226] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.50.1.47@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746433.467682] LustreError: Skipped 13 previous similar messages [1746435.470419] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.50.2.24@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746435.487931] LustreError: Skipped 76 previous similar messages [1746439.478025] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.50.8.17@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746439.495487] LustreError: Skipped 159 previous similar messages [1746447.483320] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.50.16.3@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746447.500788] LustreError: Skipped 404 previous similar messages [1746448.658158] LustreError: 66115:0:(mgc_request.c:249:do_config_log_add()) MGC10.0.10.51@o2ib7: failed processing log, type 1: rc = -5 [1746466.897577] LustreError: 137-5: fir-MDT0003_UUID: not available for connect from 10.50.9.58@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1746466.915066] LustreError: Skipped 698 previous similar messages [1746479.671008] LustreError: 66115:0:(mgc_request.c:249:do_config_log_add()) MGC10.0.10.51@o2ib7: failed processing log, type 4: rc = -110 [1746482.119522] Lustre: fir-MDT0003: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-900 [1746482.285090] Lustre: fir-MDD0003: changelog on [1746482.290506] Lustre: fir-MDT0003: in recovery but waiting for the first client to connect [1746487.128159] Lustre: fir-MDT0003: Will be in recovery for at least 2:30, or until 1295 clients reconnect [1746488.135667] Lustre: fir-MDT0003: Connection restored to (at 10.50.6.10@o2ib2) [1746488.143071] Lustre: Skipped 274 previous similar messages [1746488.635388] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.1@o2ib2) [1746488.642789] Lustre: Skipped 132 previous similar messages [1746489.649231] Lustre: fir-MDT0003: Connection restored to (at 10.50.5.49@o2ib2) [1746489.656633] Lustre: Skipped 257 previous similar messages [1746491.640050] LustreError: 66623:0:(mdt_open.c:1364:mdt_reint_open()) @@@ OPEN & CREAT not in open replay/by_fid. req@ffff8edbf0c8b600 x1672500013527168/t0(709013357627) o101->c5336eb6-8040-4@10.49.0.62@o2ib1:65/0 lens 960/3288 e 0 to 0 dl 1595971915 ref 1 fl Interpret:/4/0 rc 0/0 [1746491.652454] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.68@o2ib2) [1746491.652456] Lustre: Skipped 495 previous similar messages [1746534.228116] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.115@o2ib7) [1746534.235635] Lustre: Skipped 127 previous similar messages [1746544.601567] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.103@o2ib7) [1746544.609060] Lustre: Skipped 55 previous similar messages [1746555.822345] LustreError: 66598:0:(fld_handler.c:225:fld_local_lookup()) srv-fir-MDT0003: FLD cache range [0x0000001440000400-0x0000001480000400]:45:ost does not match requested flag 0: rc = -5 [1746555.956563] Lustre: fir-MDT0003: Recovery over after 1:09, of 1295 clients 1295 recovered and 0 were evicted. [1752377.782201] Lustre: fir-MDT0003: Connection restored to 34b6d393-8480-4 (at 10.50.7.64@o2ib2) [1752377.790903] Lustre: Skipped 36 previous similar messages [1752926.567594] Lustre: fir-MDT0003: haven't heard from client 95b4b43b-52f4-4 (at 10.51.0.14@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eca9ea19c00, cur 1595978308 expire 1595978158 last 1595978081 [1753095.592442] Lustre: fir-MDT0003: Connection restored to 0292c007-085e-4 (at 10.49.7.3@o2ib1) [1753518.963527] Lustre: fir-MDT0003: Connection restored to df89d18a-e900-4 (at 10.51.1.2@o2ib3) [1753768.590773] Lustre: fir-MDT0003: haven't heard from client 02cf9217-e850-4 (at 10.51.0.11@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eca9ea1e800, cur 1595979150 expire 1595979000 last 1595978923 [1753794.906214] Lustre: fir-MDT0003: Connection restored to f93b9579-5722-4 (at 10.51.1.5@o2ib3) [1753794.914833] Lustre: Skipped 1 previous similar message [1754060.501541] Lustre: fir-MDT0003: Connection restored to 875aa40f-95a4-4 (at 10.51.1.9@o2ib3) [1754060.510162] Lustre: Skipped 2 previous similar messages [1754332.042217] Lustre: fir-MDT0003: Connection restored to 261e0a2f-e903-4 (at 10.51.1.15@o2ib3) [1754332.050927] Lustre: Skipped 2 previous similar messages [1755102.914513] Lustre: fir-MDT0003: Connection restored to 584b3772-1e89-4 (at 10.51.1.49@o2ib3) [1755102.923213] Lustre: Skipped 3 previous similar messages [1755103.917157] Lustre: fir-MDT0003: Connection restored to 2d83c847-c2df-4 (at 10.51.1.61@o2ib3) [1755103.925857] Lustre: Skipped 26 previous similar messages [1755120.375786] Lustre: fir-MDT0003: Connection restored to 978ff576-93a7-4 (at 10.51.1.71@o2ib3) [1755120.384490] Lustre: Skipped 17 previous similar messages [1755130.992742] Lustre: fir-MDT0003: Connection restored to 9862df29-ae56-4 (at 10.51.1.60@o2ib3) [1756541.193008] Lustre: fir-MDT0003: Connection restored to d587d8a6-7073-4 (at 10.51.1.19@o2ib3) [1756541.201711] Lustre: Skipped 3 previous similar messages [1756792.872391] Lustre: fir-MDT0003: Connection restored to 0b5da860-c582-4 (at 10.51.1.4@o2ib3) [1756792.881010] Lustre: Skipped 6 previous similar messages [1761496.747185] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.65@o2ib2) [1761507.804423] Lustre: fir-MDT0003: haven't heard from client ce4c63db-6c11-4 (at 10.50.7.65@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef2d1b09400, cur 1595986889 expire 1595986739 last 1595986662 [1765893.047307] Lustre: fir-MDT0003: Connection restored to (at 10.50.12.13@o2ib2) [1765932.929614] Lustre: fir-MDT0003: haven't heard from client f2b52403-6ed7-4 (at 10.50.12.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef1ee672400, cur 1595991314 expire 1595991164 last 1595991087 [1772936.145943] Lustre: fir-MDT0003: haven't heard from client f29d7d43-7979-4 (at 10.50.8.18@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb5baccc00, cur 1595998317 expire 1595998167 last 1595998090 [1772954.621984] Lustre: fir-MDT0003: Connection restored to (at 10.50.8.18@o2ib2) [1789071.958708] LustreError: 66801:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'currentdata_200117' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 currentdata_200117' to finish migration. [1808462.573977] LustreError: 11-0: fir-MDT0002-osp-MDT0003: operation mds_statfs to node 10.0.10.53@o2ib7 failed: rc = -107 [1808462.584943] Lustre: fir-MDT0002-osp-MDT0003: Connection to fir-MDT0002 (at 10.0.10.53@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1808480.611783] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.2.34@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808480.629237] LustreError: Skipped 7 previous similar messages [1808485.105695] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.6.36@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808485.123155] LustreError: Skipped 4 previous similar messages [1808493.118338] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.3.10@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808493.135795] LustreError: Skipped 462 previous similar messages [1808509.120243] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.50.14.4@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808509.137700] LustreError: Skipped 578 previous similar messages [1808542.901861] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.49.18.34@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808542.919400] LustreError: Skipped 407 previous similar messages [1808582.130318] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1808582.140578] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [1808588.130488] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 0 seconds [1808595.130675] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.53@o2ib7: 1 seconds [1808606.911311] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.1.15@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808606.928767] LustreError: Skipped 862 previous similar messages [1808615.090567] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.53@o2ib7) [1808677.138568] Lustre: fir-MDT0002-osp-MDT0003: Connection restored to 10.0.10.53@o2ib7 (at 10.0.10.53@o2ib7) [1808743.013986] LustreError: 11-0: fir-MDT0000-osp-MDT0003: operation mds_statfs to node 10.0.10.51@o2ib7 failed: rc = -107 [1808743.024946] Lustre: fir-MDT0000-osp-MDT0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1808752.278250] Lustre: fir-MDT0000-lwp-MDT0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1808813.755907] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 10.50.1.35@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1808813.773362] LustreError: Skipped 589 previous similar messages [1808834.464459] Lustre: 66071:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1596034207/real 1596034207] req@ffff8ef3f4259b00 x1673500052610240/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1596034214 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1808834.492672] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1808841.137657] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [1808847.137854] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1808884.634120] Lustre: Evicted from MGS (at MGC10.0.10.51@o2ib7_0) after server handle changed from 0x86a53872e5fed529 to 0x8a212c1aa7522a78 [1808884.646812] Lustre: MGC10.0.10.51@o2ib7: Connection restored to MGC10.0.10.51@o2ib7_0 (at 10.0.10.51@o2ib7) [1808887.951580] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.51@o2ib7) [1808945.435625] LNet: Service thread pid 74058 was inactive for 200.01s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1808945.452738] Pid: 74058, comm: mdt01_094 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1808945.463085] Call Trace: [1808945.465735] [] ptlrpc_set_wait+0x480/0x790 [ptlrpc] [1808945.472506] [] ptlrpc_queue_wait+0x83/0x230 [ptlrpc] [1808945.479369] [] ldlm_cli_enqueue+0x3d2/0x920 [ptlrpc] [1808945.486217] [] osp_md_object_lock+0x162/0x2d0 [osp] [1808945.492974] [] lod_object_lock+0xf4/0x780 [lod] [1808945.499374] [] mdd_object_lock+0x3e/0xe0 [mdd] [1808945.505705] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [1808945.513152] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [1808945.520085] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [1808945.526483] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [1808945.533152] [] mdt_reint_rec+0x83/0x210 [mdt] [1808945.539379] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1808945.546140] [] mdt_reint+0x67/0x140 [mdt] [1808945.552016] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1808945.559158] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1808945.567046] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1808945.573566] [] kthread+0xd1/0xe0 [1808945.578658] [] ret_from_fork_nospec_begin+0xe/0x21 [1808945.585319] [] 0xffffffffffffffff [1808945.590524] LustreError: dumping log to /tmp/lustre-log.1596034325.74058 [1808954.651894] LNet: Service thread pid 66218 was inactive for 200.16s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1808954.669003] Pid: 66218, comm: mdt02_003 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1808954.679350] Call Trace: [1808954.681987] [] ptlrpc_set_wait+0x480/0x790 [ptlrpc] [1808954.688767] [] ptlrpc_queue_wait+0x83/0x230 [ptlrpc] [1808954.695634] [] ldlm_cli_enqueue+0x3d2/0x920 [ptlrpc] [1808954.702480] [] osp_md_object_lock+0x162/0x2d0 [osp] [1808954.709230] [] lod_object_lock+0xf4/0x780 [lod] [1808954.715628] [] mdd_object_lock+0x3e/0xe0 [mdd] [1808954.721952] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [1808954.729392] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [1808954.736313] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [1808954.742702] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [1808954.749375] [] mdt_reint_rec+0x83/0x210 [mdt] [1808954.755607] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [1808954.762355] [] mdt_reint+0x67/0x140 [mdt] [1808954.768228] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1808954.775358] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1808954.783249] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1808954.789765] [] kthread+0xd1/0xe0 [1808954.794852] [] ret_from_fork_nospec_begin+0xe/0x21 [1808954.801504] [] 0xffffffffffffffff [1808954.806702] LustreError: dumping log to /tmp/lustre-log.1596034334.66218 [1808959.901162] LustreError: 167-0: fir-MDT0000-lwp-MDT0003: This client was evicted by fir-MDT0000; in progress operations using this service will fail. [1808959.965575] Lustre: fir-MDT0000-lwp-MDT0003: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [1808960.113204] LNet: Service thread pid 74058 completed after 214.68s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [1808960.129538] LNet: Skipped 1 previous similar message [1816776.395229] Lustre: fir-MDT0003: haven't heard from client 0e178934-05d5-4 (at 10.50.17.38@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf9a41c00, cur 1596042156 expire 1596042006 last 1596041929 [1817105.273923] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.38@o2ib2) [1817105.281414] Lustre: Skipped 1 previous similar message [1821566.533878] Lustre: fir-MDT0003: haven't heard from client 47734007-ee62-4 (at 10.51.1.7@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edf0d255400, cur 1596046946 expire 1596046796 last 1596046719 [1821825.975233] Lustre: fir-MDT0003: Connection restored to efe9d011-afcc-4 (at 10.51.1.33@o2ib3) [1821826.763073] Lustre: fir-MDT0003: Connection restored to 4dd9e454-af76-4 (at 10.51.1.34@o2ib3) [1821827.769822] Lustre: fir-MDT0003: Connection restored to cb100460-04ad-4 (at 10.51.1.63@o2ib3) [1821827.778526] Lustre: Skipped 20 previous similar messages [1821837.675806] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [1821837.684419] Lustre: Skipped 35 previous similar messages [1821843.698010] Lustre: fir-MDT0003: Connection restored to 978ff576-93a7-4 (at 10.51.1.71@o2ib3) [1821990.013329] Lustre: fir-MDT0003: Connection restored to 9862df29-ae56-4 (at 10.51.1.60@o2ib3) [1822379.390426] Lustre: fir-MDT0003: Connection restored to e7c6eee2-5dad-4 (at 10.51.1.11@o2ib3) [1822379.399135] Lustre: Skipped 1 previous similar message [1826251.914718] Lustre: fir-MDT0003: Connection restored to 831e6972-eb25-4 (at 10.50.2.33@o2ib2) [1826251.923424] Lustre: Skipped 9 previous similar messages [1833206.550815] Lustre: fir-MDT0003: Connection restored to 831e6972-eb25-4 (at 10.50.2.33@o2ib2) [1838511.004312] Lustre: fir-MDT0003: haven't heard from client 6b1edbac-863a-4 (at 10.51.0.63@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef8fbe12400, cur 1596063890 expire 1596063740 last 1596063663 [1838511.024392] Lustre: Skipped 71 previous similar messages [1842869.202489] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [1842897.122183] Lustre: fir-MDT0003: haven't heard from client d24c2254-c738-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eca9ea1ac00, cur 1596068276 expire 1596068126 last 1596068049 [1868852.871948] Lustre: fir-MDT0003: haven't heard from client 7729831f-ffe1-4 (at 10.50.16.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf9a79000, cur 1596094231 expire 1596094081 last 1596094004 [1869044.849634] Lustre: fir-MDT0003: Connection restored to (at 10.50.16.10@o2ib2) [1887940.180750] LustreError: 67032:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'data' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 data' to finish migration. [1888053.448242] LustreError: 66621:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '01-bare' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 01-bare' to finish migration. [1888062.519461] LustreError: 67044:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '01-bare' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 01-bare' to finish migration. [1888074.881599] LustreError: 66818:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '02-ts' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 02-ts' to finish migration. [1888155.849110] LustreError: 67023:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '04-waterdown-AP-fromC' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 04-waterdown-AP-fromC' to finish migration. [1888194.321740] LustreError: 66167:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '02-optfinal' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 02-optfinal' to finish migration. [1888301.498643] LustreError: 67011:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '03-waterup-AP-ckirkO' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 03-waterup-AP-ckirkO' to finish migration. [1888405.340148] LustreError: 66818:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '02-waterdown-AP-fromO' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 02-waterdown-AP-fromO' to finish migration. [1889646.285846] LustreError: 67009:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [1905612.796801] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [1905639.930453] Lustre: fir-MDT0003: haven't heard from client 3578eaaa-7b94-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa7db400, cur 1596131017 expire 1596130867 last 1596130790 [1911539.470834] Lustre: fir-MDT0003: Client 0dbe8e53-cb7f-4 (at 10.50.17.40@o2ib2) reconnecting [1911539.471342] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.12@o2ib2) [1911539.486860] Lustre: Skipped 6 previous similar messages [1911633.970702] LustreError: 66789:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [1912086.673578] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [1912086.682203] Lustre: Skipped 7 previous similar messages [1912313.121762] Lustre: fir-MDT0003: haven't heard from client a2304d93-40e3-4 (at 10.51.1.1@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ebc20c2c000, cur 1596137690 expire 1596137540 last 1596137463 [1912404.555903] Lustre: fir-MDT0003: Client 66b0fff0-30cb-4 (at 10.51.1.34@o2ib3) reconnecting [1912404.564340] Lustre: Skipped 1 previous similar message [1912404.569687] Lustre: fir-MDT0003: Connection restored to 4dd9e454-af76-4 (at 10.51.1.34@o2ib3) [1912405.522783] Lustre: fir-MDT0003: Client b82b1576-1e10-4 (at 10.51.1.66@o2ib3) reconnecting [1912406.851117] Lustre: fir-MDT0003: Client 896e26e2-d6af-4 (at 10.51.1.9@o2ib3) reconnecting [1912406.859477] Lustre: Skipped 11 previous similar messages [1912406.864995] Lustre: fir-MDT0003: Connection restored to 875aa40f-95a4-4 (at 10.51.1.9@o2ib3) [1912406.873621] Lustre: Skipped 12 previous similar messages [1912435.000506] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.1.5@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1912435.017894] LustreError: Skipped 1473 previous similar messages [1912480.252541] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [1912480.261162] Lustre: Skipped 3 previous similar messages [1912490.422200] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 10.51.1.47@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1912490.439661] LustreError: Skipped 2 previous similar messages [1912512.399433] Lustre: fir-MDT0003: Client 70b07298-c120-4 (at 10.51.1.13@o2ib3) reconnecting [1912512.407874] Lustre: Skipped 3 previous similar messages [1912512.413314] Lustre: fir-MDT0003: Connection restored to 60aeb526-7f18-4 (at 10.51.1.13@o2ib3) [1912518.320956] Lustre: fir-MDT0003: Client 76ed293e-524e-4 (at 10.51.1.42@o2ib3) reconnecting [1912613.290442] Lustre: fir-MDT0003: Client bdc9e2c8-17ba-4 (at 10.51.0.72@o2ib3) reconnecting [1912613.298903] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.72@o2ib3) [1912613.306300] Lustre: Skipped 1 previous similar message [1913075.117968] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.11@o2ib3) [1922661.177312] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [1926437.508207] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1926731.377338] Lustre: fir-MDT0003: Client a209d553-7ee4-4 (at 10.51.1.45@o2ib3) reconnecting [1926731.385823] Lustre: fir-MDT0003: Connection restored to 83fff3d8-9432-4 (at 10.51.1.45@o2ib3) [1926735.577683] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.1.5@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1926743.605525] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.1.37@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1926743.623118] LustreError: Skipped 20 previous similar messages [1926760.149671] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.0.72@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1926760.167124] LustreError: Skipped 14 previous similar messages [1926792.521357] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 10.51.1.54@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1926792.538810] LustreError: Skipped 109 previous similar messages [1926806.725998] Lustre: fir-MDT0003: Client 6855dbf6-3c80-4 (at 10.51.1.72@o2ib3) reconnecting [1926806.734466] Lustre: fir-MDT0003: Connection restored to b38b6203-c277-4 (at 10.51.1.72@o2ib3) [1926810.586952] Lustre: fir-MDT0003: Connection restored to 978ff576-93a7-4 (at 10.51.1.71@o2ib3) [1926810.595655] Lustre: Skipped 3 previous similar messages [1926810.821702] Lustre: fir-MDT0003: Client 175dabb9-3409-4 (at 10.51.1.5@o2ib3) reconnecting [1926810.830062] Lustre: Skipped 6 previous similar messages [1926815.329133] Lustre: fir-MDT0003: Connection restored to bc531068-74a4-4 (at 10.51.1.10@o2ib3) [1926815.337839] Lustre: Skipped 24 previous similar messages [1926818.870649] Lustre: fir-MDT0003: Client 79ffc9a3-50eb-4 (at 10.51.1.37@o2ib3) reconnecting [1926818.879090] Lustre: Skipped 39 previous similar messages [1926823.875458] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [1926823.884078] Lustre: Skipped 30 previous similar messages [1926835.415644] Lustre: fir-MDT0003: Client bdc9e2c8-17ba-4 (at 10.51.0.72@o2ib3) reconnecting [1926835.424091] Lustre: Skipped 23 previous similar messages [1926844.736849] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1926844.756822] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1926941.197594] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.1.66@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1926941.215048] LustreError: Skipped 48 previous similar messages [1927007.663368] Lustre: fir-MDT0003: Client 8646c5c9-c625-4 (at 10.51.1.55@o2ib3) reconnecting [1927007.671810] Lustre: Skipped 8 previous similar messages [1927007.677240] Lustre: fir-MDT0003: Connection restored to 319232aa-b7f8-4 (at 10.51.1.55@o2ib3) [1927007.685946] Lustre: Skipped 19 previous similar messages [1927328.765410] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1927328.785383] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1927712.885092] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1927712.905068] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1927939.016031] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1927939.035999] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1928423.462547] Lustre: fir-MDT0003: Client 20089e30-71d9-4 (at 10.51.1.17@o2ib3) reconnecting [1928423.470994] Lustre: Skipped 23 previous similar messages [1928423.476512] Lustre: fir-MDT0003: Connection restored to be8da727-2029-4 (at 10.51.1.17@o2ib3) [1928423.485211] Lustre: Skipped 23 previous similar messages [1928426.141721] LNetError: 109327:0:(o2iblnd_cb.c:2962:kiblnd_rejected()) 10.0.10.235@o2ib7 rejected: o2iblnd fatal error [1928429.531075] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.232@o2ib7: 0 seconds [1928429.541422] LNet: 66031:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 1 previous similar message [1928441.157904] Lustre: fir-MDT0003: Client 4ecb334f-80fe-4 (at 10.51.1.57@o2ib3) reconnecting [1928441.166349] Lustre: Skipped 3 previous similar messages [1928441.171782] Lustre: fir-MDT0003: Connection restored to 30410dde-0728-4 (at 10.51.1.57@o2ib3) [1928441.180485] Lustre: Skipped 3 previous similar messages [1928455.450049] LustreError: 137-5: fir-MDT0002_UUID: not available for connect from 10.51.0.13@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1928455.467513] LustreError: Skipped 58 previous similar messages [1928505.555447] LustreError: 137-5: fir-MDT0000_UUID: not available for connect from 10.51.0.65@o2ib3 (no target). If you are running an HA pair check that the target is mounted on the other server. [1928827.676539] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1928827.696510] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1929170.440104] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1929170.460074] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1929296.697908] Lustre: fir-MDT0003: Connection restored to a94d2ded-b7db-4 (at 10.51.0.15@o2ib3) [1929296.706613] Lustre: Skipped 1 previous similar message [1929319.126055] Lustre: fir-MDT0003: Connection restored to 5d01f42c-8579-4 (at 10.51.0.12@o2ib3) [1929541.129603] Lustre: fir-MDT0003: Connection restored to b232156a-01ab-4 (at 10.51.0.67@o2ib3) [1929549.228553] Lustre: fir-MDT0003: Connection restored to d601a89e-9f11-4 (at 10.51.0.68@o2ib3) [1929568.295763] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.65@o2ib3) [1929722.123901] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [1929760.192324] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.66@o2ib3) [1929843.888528] Lustre: fir-MDT0003: Connection restored to 2390729b-b58f-4 (at 10.51.1.65@o2ib3) [1930305.337545] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1930305.357519] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1930435.471465] Lustre: fir-MDT0003: Connection restored to f93b9579-5722-4 (at 10.51.1.5@o2ib3) [1930435.480087] Lustre: Skipped 7 previous similar messages [1930438.001218] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1930438.021194] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1930701.131454] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1930701.151422] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1930904.604508] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1930904.624483] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1931118.945162] Lustre: fir-MDT0003: Connection restored to 6f094883-b7cb-4 (at 10.51.1.23@o2ib3) [1931118.953866] Lustre: Skipped 39 previous similar messages [1931179.862240] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1931179.882211] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1931736.435214] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1931736.455180] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1931865.151748] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1931865.171721] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1931990.004411] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1931990.024385] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1932285.064809] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1932285.084781] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1932454.711149] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1932454.731139] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 3 previous similar messages [1932728.066297] Lustre: fir-MDT0003: Connection restored to 4dd9e454-af76-4 (at 10.51.1.34@o2ib3) [1932728.075001] Lustre: Skipped 1 previous similar message [1932900.911988] Lustre: fir-MDT0003: Connection restored to 30410dde-0728-4 (at 10.51.1.57@o2ib3) [1932900.920687] Lustre: Skipped 8 previous similar messages [1932929.190488] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1932929.210460] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1933664.832324] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1933664.852297] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 7 previous similar messages [1935015.249371] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1935015.269347] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 9 previous similar messages [1935234.795077] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1935234.815045] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 1 previous similar message [1935526.340785] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1935526.360755] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) Skipped 3 previous similar messages [1935839.621146] Lustre: fir-MDT0003: Connection restored to 9862df29-ae56-4 (at 10.51.1.60@o2ib3) [1935839.629855] Lustre: Skipped 10 previous similar messages [1936173.584224] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1936173.604194] LustreError: 66192:0:(out_handler.c:915:out_tx_end()) Skipped 7 previous similar messages [1936854.458411] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1936854.478382] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 9 previous similar messages [1937662.854983] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1937662.874951] LustreError: 66997:0:(out_handler.c:915:out_tx_end()) Skipped 11 previous similar messages [1938311.538168] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1938311.558144] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 5 previous similar messages [1939337.938752] LustreError: 66191:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1939337.958722] LustreError: 66191:0:(out_handler.c:915:out_tx_end()) Skipped 9 previous similar messages [1939944.625584] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1939944.645562] LustreError: 66964:0:(out_handler.c:915:out_tx_end()) Skipped 5 previous similar messages [1940666.456738] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) fir-MDT0003-osd: undo for /tmp/rpmbuild-lustre-sthiell-X0XCFmG5/BUILD/lustre-2.12.5_2_g8ac362a/lustre/ptlrpc/../../lustre/target/out_handler.c:454: rc = -524 [1940666.476707] LustreError: 69033:0:(out_handler.c:915:out_tx_end()) Skipped 9 previous similar messages [1941792.534770] LustreError: 66828:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1941794.414618] LustreError: 66759:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=618' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=618' to finish migration. [1941821.149477] LustreError: 66690:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1941853.135551] LustreError: 66664:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1941853.151535] LustreError: 66664:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 1 previous similar message [1941865.779826] LustreError: 66166:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1941888.810134] LustreError: 66659:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1941888.826120] LustreError: 66659:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 1 previous similar message [1941905.778007] LustreError: 66723:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=618' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=618' to finish migration. [1941905.793987] LustreError: 66723:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 2 previous similar messages [1941942.906837] LustreError: 66819:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1942033.808021] LustreError: 66813:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: 'seed=619' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 seed=619' to finish migration. [1942033.824004] LustreError: 66813:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 7 previous similar messages [1943418.116558] LustreError: 66167:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [1943418.132712] LustreError: 66167:0:(mdd_dir.c:4496:mdd_migrate()) Skipped 9 previous similar messages [1975738.872771] list passed to list_sort() too long for efficiency [1992988.595050] LustreError: 66701:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2000328.015441] Lustre: fir-MDT0003: Connection restored to 55613a18-5a1e-4 (at 10.51.1.1@o2ib3) [2002909.684398] Lustre: fir-MDT0003: haven't heard from client a94d2ded-b7db-4 (at 10.51.0.15@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef7e92ebc00, cur 1596228284 expire 1596228134 last 1596228057 [2003507.700906] Lustre: fir-MDT0003: haven't heard from client b472b069-eb19-4 (at 10.51.0.13@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf9a7a800, cur 1596228882 expire 1596228732 last 1596228655 [2003584.805121] Lustre: fir-MDT0003: Connection restored to a94d2ded-b7db-4 (at 10.51.0.15@o2ib3) [2003894.334836] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.13@o2ib3) [2005852.963903] Lustre: fir-MDT0003: Connection restored to 2f626d11-3749-4 (at 10.51.0.17@o2ib3) [2007457.798646] LustreError: 67048:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.jobdb2' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .jobdb2' to finish migration. [2007460.064504] LustreError: 66818:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2007544.377973] LustreError: 66630:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2009181.857295] Lustre: fir-MDT0003: haven't heard from client 6d5c27e9-5828-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebe0779000, cur 1596234556 expire 1596234406 last 1596234329 [2009230.081892] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [2011229.002015] LustreError: 66614:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2024813.381962] Lustre: fir-MDT0003: Connection restored to (at 10.50.12.13@o2ib2) [2033560.532232] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaa16b9400, cur 1596258934 expire 1596258784 last 1596258707 [2037015.994066] LustreError: 96025:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.49.24.23@o2ib1 arrived at 1596262389 with bad export cookie 3986339534432481629 [2038561.225563] LustreError: 66664:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2040472.823578] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2040699.742982] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb47f64800, cur 1596266073 expire 1596265923 last 1596265846 [2042555.796520] Lustre: fir-MDT0003: haven't heard from client 385ee6a2-34c8-4 (at 10.50.7.65@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf784b400, cur 1596267929 expire 1596267779 last 1596267702 [2042561.951231] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.65@o2ib2) [2051215.896223] Lustre: fir-MDT0003: Connection restored to (at 10.49.7.4@o2ib1) [2055845.186663] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2056072.190776] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbf0615400, cur 1596281445 expire 1596281295 last 1596281218 [2060697.352864] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2060924.319051] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed7e0a41000, cur 1596286297 expire 1596286147 last 1596286070 [2070053.619082] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2070280.580807] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb50f76c00, cur 1596295653 expire 1596295503 last 1596295426 [2074829.754113] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2075056.717748] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb7f656400, cur 1596300429 expire 1596300279 last 1596300202 [2082185.958672] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2082412.924952] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb7f659800, cur 1596307785 expire 1596307635 last 1596307558 [2084166.025492] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2084392.980365] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed90f61ac00, cur 1596309765 expire 1596309615 last 1596309538 [2086662.074974] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2086889.049495] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecf296f9000, cur 1596312261 expire 1596312111 last 1596312034 [2087134.104371] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2087361.060205] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed9b0bfe800, cur 1596312733 expire 1596312583 last 1596312506 [2092050.213607] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2092277.192639] Lustre: fir-MDT0003: haven't heard from client 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb367f1c00, cur 1596317649 expire 1596317499 last 1596317422 [2120476.427367] LustreError: 66166:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2120541.441437] LustreError: 66698:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2169557.080657] LustreError: 66641:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2177440.595949] Lustre: fir-MDT0003: haven't heard from client 46250fb4-36ab-4 (at 10.50.7.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeb5260ec00, cur 1596402810 expire 1596402660 last 1596402583 [2177522.951413] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.20@o2ib2) [2180247.059125] LustreError: 66650:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2180281.013726] LustreError: 66734:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2182263.331762] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [2182294.725599] Lustre: fir-MDT0003: haven't heard from client 0c34e622-a7ef-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebbab09800, cur 1596407664 expire 1596407514 last 1596407437 [2186183.827803] Lustre: fir-MDT0003: haven't heard from client 1fc12deb-ca62-4 (at 10.49.31.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebfe9d3c00, cur 1596411553 expire 1596411403 last 1596411326 [2205353.430135] Lustre: 66188:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [2205353.929325] Lustre: 66745:0:(mdd_device.c:1811:mdd_changelog_clear()) fir-MDD0003: Failure to clear the changelog for user 1: -22 [2205353.941155] Lustre: 66745:0:(mdd_device.c:1811:mdd_changelog_clear()) Skipped 502 previous similar messages [2206947.069388] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [2206953.403927] Lustre: fir-MDT0003: haven't heard from client e7c9f6b0-2b2f-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edb7f65b800, cur 1596432322 expire 1596432172 last 1596432095 [2210853.780812] LustreError: 66674:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2235913.605230] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.9@o2ib2) [2235929.251997] Lustre: fir-MDT0003: haven't heard from client 45dde4d4-6ec4-4 (at 10.50.7.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfee8cc00, cur 1596461297 expire 1596461147 last 1596461070 [2242435.397488] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2242441.436170] Lustre: fir-MDT0003: haven't heard from client 8746bf8c-2bf8-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeaa16be800, cur 1596467809 expire 1596467659 last 1596467582 [2243682.471210] Lustre: fir-MDT0003: haven't heard from client 5d01f42c-8579-4 (at 10.51.0.12@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbde659400, cur 1596469050 expire 1596468900 last 1596468823 [2244586.650459] Lustre: fir-MDT0003: Connection restored to 5d01f42c-8579-4 (at 10.51.0.12@o2ib3) [2244603.756319] Lustre: fir-MDT0003: Connection restored to 67703ce6-337f-4 (at 10.51.0.16@o2ib3) [2244840.504516] Lustre: fir-MDT0003: haven't heard from client d601a89e-9f11-4 (at 10.51.0.68@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfeb3ec00, cur 1596470208 expire 1596470058 last 1596469981 [2245358.547176] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.66@o2ib3) [2245373.041677] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2245379.520300] Lustre: fir-MDT0003: haven't heard from client 90197248-3c24-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebbab09800, cur 1596470747 expire 1596470597 last 1596470520 [2245379.540367] Lustre: Skipped 1 previous similar message [2245660.172259] Lustre: fir-MDT0003: Connection restored to d601a89e-9f11-4 (at 10.51.0.68@o2ib3) [2246220.029411] Lustre: fir-MDT0003: Connection restored to 472ba92c-8ede-4 (at 10.51.0.64@o2ib3) [2249348.484346] Lustre: fir-MDT0003: Connection restored to (at 10.51.0.63@o2ib3) [2256760.830579] Lustre: fir-MDT0003: haven't heard from client 63504435-4675-4 (at 10.49.18.29@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef61cfe1400, cur 1596482128 expire 1596481978 last 1596481901 [2258592.879496] Lustre: fir-MDT0003: haven't heard from client b232156a-01ab-4 (at 10.51.0.67@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec6c9cca000, cur 1596483960 expire 1596483810 last 1596483733 [2258974.074285] Lustre: fir-MDT0003: Connection restored to b232156a-01ab-4 (at 10.51.0.67@o2ib3) [2259661.465074] Lustre: fir-MDT0003: Connection restored to (at 10.49.18.29@o2ib1) [2259932.721193] Lustre: fir-MDT0003: Connection restored to 28ea0339-390e-4 (at 10.51.0.2@o2ib3) [2259958.980910] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [2259995.918250] Lustre: fir-MDT0003: haven't heard from client 4f4944cb-ce29-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efbfa7da000, cur 1596485363 expire 1596485213 last 1596485136 [2260396.986309] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2263570.603284] Lustre: fir-MDT0003: Connection restored to (at 10.49.31.8@o2ib1) [2263734.229123] Lustre: fir-MDT0003: Connection restored to cc158cf2-ce00-4 (at 10.50.5.13@o2ib2) [2263867.463743] LustreError: 66723:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2268170.593412] Lustre: fir-MDT0003: Connection restored to 1d40eac0-6f78-4 (at 10.49.24.23@o2ib1) [2268219.142825] Lustre: fir-MDT0003: haven't heard from client 78e4859a-6bea-4 (at 10.49.24.23@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edce3fd1800, cur 1596493586 expire 1596493436 last 1596493359 [2284652.586146] Lustre: fir-MDT0003: haven't heard from client c9b6379b-49b5-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef998a2c800, cur 1596510019 expire 1596509869 last 1596509792 [2284659.152478] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2337167.107283] Lustre: fir-MDT0003: haven't heard from client a8d2a27b-3059-4 (at 10.51.1.29@o2ib3) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eddd434bc00, cur 1596562532 expire 1596562382 last 1596562305 [2342041.829038] Lustre: fir-MDT0003: Client c5336eb6-8040-4 (at 10.49.0.62@o2ib1) reconnecting [2342041.837481] Lustre: Skipped 1 previous similar message [2342041.842846] Lustre: fir-MDT0003: Connection restored to c5336eb6-8040-4 (at 10.49.0.62@o2ib1) [2342126.070353] Lustre: fir-MDT0003: Client c5336eb6-8040-4 (at 10.49.0.62@o2ib1) reconnecting [2342126.078843] Lustre: fir-MDT0003: Connection restored to c5336eb6-8040-4 (at 10.49.0.62@o2ib1) [2342545.339705] Lustre: fir-MDT0003: Client c5336eb6-8040-4 (at 10.49.0.62@o2ib1) reconnecting [2342545.348190] Lustre: fir-MDT0003: Connection restored to c5336eb6-8040-4 (at 10.49.0.62@o2ib1) [2343087.848551] Lustre: fir-MDT0003: Client c5336eb6-8040-4 (at 10.49.0.62@o2ib1) reconnecting [2343087.857023] Lustre: fir-MDT0003: Connection restored to c5336eb6-8040-4 (at 10.49.0.62@o2ib1) [2347766.872396] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2347783.401305] Lustre: fir-MDT0003: haven't heard from client 7685c4b0-316d-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec672b38800, cur 1596573148 expire 1596572998 last 1596572921 [2348242.952510] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2348260.413429] Lustre: fir-MDT0003: haven't heard from client c2f5e97f-6537-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfebde000, cur 1596573625 expire 1596573475 last 1596573398 [2353196.725643] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.35@o2ib1) [2353219.549069] Lustre: fir-MDT0003: haven't heard from client 83e815a9-e378-4 (at 10.49.26.35@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ef5a7ed4000, cur 1596578584 expire 1596578434 last 1596578357 [2361246.763378] Lustre: fir-MDT0003: haven't heard from client 0dbe8e53-cb7f-4 (at 10.50.17.40@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebdb0ebc00, cur 1596586611 expire 1596586461 last 1596586384 [2362040.125347] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.40@o2ib2) [2362536.798645] Lustre: fir-MDT0003: haven't heard from client 0dbe8e53-cb7f-4 (at 10.50.17.40@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eeab8611400, cur 1596587901 expire 1596587751 last 1596587674 [2363671.072294] Lustre: fir-MDT0003: Client 4a749215-9cea-4 (at 10.50.2.49@o2ib2) reconnecting [2363671.080768] Lustre: fir-MDT0003: Connection restored to (at 10.50.2.49@o2ib2) [2371543.056828] Lustre: fir-MDT0003: haven't heard from client bcc78304-8fe3-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec8f50dc000, cur 1596596907 expire 1596596757 last 1596596680 [2408301.100031] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2412329.849446] Lustre: fir-MDT0003: Connection restored to (at 10.50.9.39@o2ib2) [2412338.232273] Lustre: fir-MDT0003: haven't heard from client 92dec773-047a-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eea42f01c00, cur 1596637701 expire 1596637551 last 1596637474 [2416046.337941] Lustre: fir-MDT0003: haven't heard from client d066f155-3da9-4 (at 10.50.9.39@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edf0d253c00, cur 1596641409 expire 1596641259 last 1596641182 [2428176.611694] LustreError: 66771:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2428980.706810] Lustre: fir-MDT0003: Connection restored to a1797fe3-23f9-4 (at 10.51.1.29@o2ib3) [2429251.679658] LNetError: 66031:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds [2429251.690004] LNetError: 66031:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.3@o2ib7 (108): c: 7, oc: 0, rc: 8 [2429370.718367] Lustre: fir-MDT0003: haven't heard from client 4ddbe5f4-ca9d-2223-4b58-9a425501efd0 (at 10.0.10.3@o2ib7) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8edbfea5b800, cur 1596654733 expire 1596654583 last 1596654506 [2429546.167503] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.3@o2ib7) [2429754.285327] LNet: Service thread pid 67021 was inactive for 200.39s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [2429754.302440] Pid: 67021, comm: mdt02_090 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2429754.312809] Call Trace: [2429754.315446] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [2429754.322583] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2429754.329863] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2429754.336741] [] osp_md_object_lock+0x162/0x2d0 [osp] [2429754.343489] [] lod_object_lock+0xf4/0x780 [lod] [2429754.349916] [] mdd_object_lock+0x3e/0xe0 [mdd] [2429754.356229] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2429754.363666] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2429754.370595] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2429754.376995] [] mdt_reint_migrate+0x563/0x11d0 [mdt] [2429754.383738] [] mdt_reint_rec+0x83/0x210 [mdt] [2429754.389976] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2429754.396720] [] mdt_reint+0x67/0x140 [mdt] [2429754.402611] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2429754.409739] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2429754.417650] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2429754.424162] [] kthread+0xd1/0xe0 [2429754.429249] [] ret_from_fork_nospec_begin+0xe/0x21 [2429754.435902] [] 0xffffffffffffffff [2429754.441098] LustreError: dumping log to /tmp/lustre-log.1596655116.67021 [2429754.450046] Pid: 66806, comm: mdt02_075 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2429754.460404] Call Trace: [2429754.463040] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [2429754.470158] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2429754.477451] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2429754.484302] [] osp_md_object_lock+0x162/0x2d0 [osp] [2429754.491058] [] lod_object_lock+0xf4/0x780 [lod] [2429754.497457] [] mdd_object_lock+0x3e/0xe0 [mdd] [2429754.503806] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2429754.511246] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2429754.518175] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2429754.524591] [] mdt_reint_migrate+0x563/0x11d0 [mdt] [2429754.531351] [] mdt_reint_rec+0x83/0x210 [mdt] [2429754.537576] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2429754.544326] [] mdt_reint+0x67/0x140 [mdt] [2429754.550203] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2429754.557343] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2429754.565242] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2429754.571778] [] kthread+0xd1/0xe0 [2429754.576872] [] ret_from_fork_nospec_begin+0xe/0x21 [2429754.583521] [] 0xffffffffffffffff [2429772.205851] LNet: Service thread pid 67018 was inactive for 200.03s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [2429772.222984] LNet: Skipped 1 previous similar message [2429772.228132] Pid: 67018, comm: mdt03_054 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2429772.238497] Call Trace: [2429772.241157] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [2429772.248273] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2429772.255577] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2429772.262440] [] osp_md_object_lock+0x162/0x2d0 [osp] [2429772.269200] [] lod_object_lock+0xf4/0x780 [lod] [2429772.275615] [] mdd_object_lock+0x3e/0xe0 [mdd] [2429772.281944] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2429772.289380] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2429772.296296] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2429772.302694] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [2429772.309363] [] mdt_reint_rec+0x83/0x210 [mdt] [2429772.315589] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2429772.322332] [] mdt_reint+0x67/0x140 [mdt] [2429772.328224] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2429772.335375] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2429772.343271] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2429772.349796] [] kthread+0xd1/0xe0 [2429772.354885] [] ret_from_fork_nospec_begin+0xe/0x21 [2429772.361547] [] 0xffffffffffffffff [2429772.366743] LustreError: dumping log to /tmp/lustre-log.1596655134.67018 [2429779.374062] LNet: Service thread pid 66763 was inactive for 200.61s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [2429779.391173] Pid: 66763, comm: mdt03_036 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2429779.401521] Call Trace: [2429779.404163] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [2429779.411303] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2429779.418582] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2429779.425468] [] osp_md_object_lock+0x162/0x2d0 [osp] [2429779.432233] [] lod_object_lock+0xf4/0x780 [lod] [2429779.438627] [] mdd_object_lock+0x3e/0xe0 [mdd] [2429779.444956] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2429779.452392] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2429779.459307] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2429779.465739] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [2429779.472395] [] mdt_reint_rec+0x83/0x210 [mdt] [2429779.478618] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2429779.485360] [] mdt_reint+0x67/0x140 [mdt] [2429779.491238] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2429779.498371] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2429779.506282] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2429779.512790] [] kthread+0xd1/0xe0 [2429779.517878] [] ret_from_fork_nospec_begin+0xe/0x21 [2429779.524527] [] 0xffffffffffffffff [2429779.529719] LustreError: dumping log to /tmp/lustre-log.1596655141.66763 [2429779.537197] Pid: 66223, comm: mdt00_004 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2429779.547541] Call Trace: [2429779.550182] [] ldlm_completion_ast+0x430/0x860 [ptlrpc] [2429779.557316] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2429779.564610] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2429779.571468] [] osp_md_object_lock+0x162/0x2d0 [osp] [2429779.578210] [] lod_object_lock+0xf4/0x780 [lod] [2429779.584604] [] mdd_object_lock+0x3e/0xe0 [mdd] [2429779.590906] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2429779.598343] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2429779.605260] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2429779.611655] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [2429779.618312] [] mdt_reint_rec+0x83/0x210 [mdt] [2429779.624535] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2429779.631278] [] mdt_reint+0x67/0x140 [mdt] [2429779.637153] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2429779.644277] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2429779.652166] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2429779.658665] [] kthread+0xd1/0xe0 [2429779.663751] [] ret_from_fork_nospec_begin+0xe/0x21 [2429779.670410] [] 0xffffffffffffffff [2429782.958160] LNet: Service thread pid 66703 was inactive for 200.26s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429782.971202] LustreError: dumping log to /tmp/lustre-log.1596655145.66703 [2429786.030258] LNet: Service thread pid 66641 was inactive for 200.38s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429786.043292] LustreError: dumping log to /tmp/lustre-log.1596655148.66641 [2429802.926742] LNet: Service thread pid 67094 was inactive for 200.01s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429802.939780] LustreError: dumping log to /tmp/lustre-log.1596655165.67094 [2429806.511847] LNet: Service thread pid 66818 was inactive for 200.45s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429806.524889] LustreError: dumping log to /tmp/lustre-log.1596655168.66818 [2429844.911979] LNet: Service thread pid 66729 was inactive for 200.19s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429844.925023] LustreError: dumping log to /tmp/lustre-log.1596655207.66729 [2429853.888234] Lustre: fir-MDT0000-osp-MDT0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [2429853.904409] LustreError: 67021:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596654916, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ee33d7398c0/0x375252c3e6b082cc lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 21 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb1215bf expref: -99 pid: 67021 timeout: 0 lvb_type: 0 [2429853.944666] LustreError: 67021:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 1 previous similar message [2429853.955967] Lustre: fir-MDT0000-osp-MDT0003: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [2429872.175775] LustreError: 67018:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596654934, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ef59bbdde80/0x375252c3e6da551b lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 21 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb14d6a4 expref: -99 pid: 67018 timeout: 0 lvb_type: 0 [2429878.756971] LustreError: 66763:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596654941, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ef38a2c8240/0x375252c3e6e92a9b lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 21 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb154d73 expref: -99 pid: 66763 timeout: 0 lvb_type: 0 [2429882.691087] LustreError: 66703:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596654944, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ef0d4323600/0x375252c3e6f320fc lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 21 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb158423 expref: -99 pid: 66703 timeout: 0 lvb_type: 0 [2429882.731371] LustreError: 66703:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 1 previous similar message [2429883.825111] LNet: Service thread pid 67041 was inactive for 236.13s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429883.838180] LustreError: dumping log to /tmp/lustre-log.1596655246.67041 [2429885.873172] LustreError: dumping log to /tmp/lustre-log.1596655248.66825 [2429902.915673] LustreError: 67094:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596654965, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ef37efbc5c0/0x375252c3e727f684 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 22 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb343f6d expref: -99 pid: 67094 timeout: 0 lvb_type: 0 [2429902.955945] LustreError: 67094:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 1 previous similar message [2429917.618098] LNet: Service thread pid 66725 was inactive for 200.53s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2429917.631133] LNet: Skipped 1 previous similar message [2429917.636283] LustreError: dumping log to /tmp/lustre-log.1596655279.66725 [2429920.690190] LustreError: dumping log to /tmp/lustre-log.1596655282.66616 [2429944.715896] LustreError: 66729:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596655006, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ec1346c5580/0x375252c3e7a3f102 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 22 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb746646 expref: -99 pid: 66729 timeout: 0 lvb_type: 0 [2429944.756169] LustreError: 66729:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 1 previous similar message [2429946.802958] LustreError: dumping log to /tmp/lustre-log.1596655309.66687 [2430007.732734] LNet: Service thread pid 66744 was inactive for 200.24s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [2430007.745777] LNet: Skipped 2 previous similar messages [2430007.751017] LustreError: dumping log to /tmp/lustre-log.1596655370.66744 [2430017.081029] LustreError: 66725:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596655079, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8edb38be6540/0x375252c3e8265e7d lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 23 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb7ac817 expref: -99 pid: 66725 timeout: 0 lvb_type: 0 [2430017.121327] LustreError: 66725:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 2 previous similar messages [2430040.501691] LustreError: dumping log to /tmp/lustre-log.1596655402.66824 [2430105.016571] LNet: Service thread pid 67275 was inactive for 362.50s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [2430105.033703] LNet: Skipped 1 previous similar message [2430105.038850] Pid: 67275, comm: mdt03_074 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2430105.049194] Call Trace: [2430105.051830] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [2430105.058953] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2430105.066247] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2430105.073096] [] osp_md_object_lock+0x162/0x2d0 [osp] [2430105.079840] [] lod_object_lock+0xf4/0x780 [lod] [2430105.086258] [] mdd_object_lock+0x3e/0xe0 [mdd] [2430105.092578] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2430105.100014] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2430105.106931] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2430105.113326] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [2430105.119983] [] mdt_reint_rec+0x83/0x210 [mdt] [2430105.126207] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2430105.132949] [] mdt_reint+0x67/0x140 [mdt] [2430105.138825] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2430105.145959] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2430105.153853] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2430105.160361] [] kthread+0xd1/0xe0 [2430105.165450] [] ret_from_fork_nospec_begin+0xe/0x21 [2430105.172097] [] 0xffffffffffffffff [2430105.177291] LustreError: dumping log to /tmp/lustre-log.1596655467.67275 [2430107.494654] LustreError: 66744:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596655169, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ecd593172c0/0x375252c3e8e3ac33 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 26 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb80c3e3 expref: -99 pid: 66744 timeout: 0 lvb_type: 0 [2430107.534911] LustreError: 66744:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 4 previous similar messages [2430149.027869] Lustre: 66621:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ee476a30000 x1672188697251072/t0(0) o36->2ef10909-6964-7c19-1207-151fcbf9efdb@10.0.10.4@o2ib7:391/0 lens 712/2888 e 23 to 0 dl 1596655516 ref 2 fl Interpret:/0/0 rc 0/0 [2430154.897909] Lustre: fir-MDT0003: Client 2ef10909-6964-7c19-1207-151fcbf9efdb (at 10.0.10.4@o2ib7) reconnecting [2430154.908120] Lustre: fir-MDT0003: Connection restored to (at 10.0.10.4@o2ib7) [2430167.545412] Lustre: 66634:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ef19216c800 x1672354011221376/t0(0) o36->647172d5-3da4-4@10.50.4.8@o2ib2:409/0 lens 552/2888 e 24 to 0 dl 1596655534 ref 2 fl Interpret:/0/0 rc 0/0 [2430167.572678] Lustre: 66634:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message [2430173.237839] Lustre: fir-MDT0003: Client 647172d5-3da4-4 (at 10.50.4.8@o2ib2) reconnecting [2430173.246232] Lustre: fir-MDT0003: Connection restored to (at 10.50.4.8@o2ib2) [2430174.297608] Lustre: 113975:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ec971a3c800 x1672346794256832/t0(0) o36->0c7d624a-f523-4@10.50.10.36@o2ib2:416/0 lens 520/2888 e 23 to 0 dl 1596655541 ref 2 fl Interpret:/0/0 rc 0/0 [2430177.561705] Lustre: 67097:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8eed49161680 x1672399316660928/t0(0) o36->4f9d968b-f0dc-4@10.50.17.48@o2ib2:419/0 lens 536/2888 e 24 to 0 dl 1596655544 ref 2 fl Interpret:/0/0 rc 0/0 [2430177.589135] Lustre: 67097:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message [2430179.842807] Lustre: fir-MDT0003: Client c4d9031c-85fb-4 (at 10.50.7.65@o2ib2) reconnecting [2430183.764108] Lustre: fir-MDT0003: Client 4f9d968b-f0dc-4 (at 10.50.17.48@o2ib2) reconnecting [2430183.772645] Lustre: Skipped 1 previous similar message [2430183.777990] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.48@o2ib2) [2430183.785476] Lustre: Skipped 2 previous similar messages [2430183.865882] LNet: Service thread pid 66810 was inactive for 437.06s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [2430183.882995] Pid: 66810, comm: mdt02_077 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2430183.893346] Call Trace: [2430183.895987] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [2430183.903116] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2430183.910406] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2430183.917266] [] osp_md_object_lock+0x162/0x2d0 [osp] [2430183.924012] [] lod_object_lock+0xf4/0x780 [lod] [2430183.930407] [] mdd_object_lock+0x3e/0xe0 [mdd] [2430183.936718] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2430183.944169] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2430183.951086] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2430183.957482] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [2430183.964139] [] mdt_reint_rec+0x83/0x210 [mdt] [2430183.970364] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2430183.977106] [] mdt_reint+0x67/0x140 [mdt] [2430183.982981] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2430183.990116] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2430183.998010] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2430184.004509] [] kthread+0xd1/0xe0 [2430184.009598] [] ret_from_fork_nospec_begin+0xe/0x21 [2430184.016246] [] 0xffffffffffffffff [2430184.021453] LustreError: dumping log to /tmp/lustre-log.1596655546.66810 [2430187.924996] Lustre: fir-MDT0000-osp-MDT0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [2430187.941169] LustreError: 67211:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596655250, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8efb49ebd100/0x375252c3e990c065 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 27 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb8650b2 expref: -99 pid: 67211 timeout: 0 lvb_type: 0 [2430187.981423] LustreError: 67211:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 2 previous similar messages [2430198.586321] Lustre: 67287:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ef39eda5a00 x1672583334981696/t0(0) o36->966ed71b-2e25-4@10.49.27.15@o2ib1:440/0 lens 552/2888 e 11 to 0 dl 1596655565 ref 2 fl Interpret:/0/0 rc 0/0 [2430198.613755] Lustre: 67287:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message [2430203.977264] Lustre: fir-MDT0003: Client 966ed71b-2e25-4 (at 10.49.27.15@o2ib1) reconnecting [2430203.985793] Lustre: Skipped 1 previous similar message [2430203.991144] Lustre: fir-MDT0003: Connection restored to (at 10.49.27.15@o2ib1) [2430203.998626] Lustre: Skipped 2 previous similar messages [2430239.484506] Lustre: 66645:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ecbdfe20850 x1672339313538880/t0(0) o36->b0c2b6ec-d912-4@10.50.0.62@o2ib2:481/0 lens 512/2888 e 7 to 0 dl 1596655606 ref 2 fl Interpret:/0/0 rc 0/0 [2430239.511778] Lustre: 66645:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message [2430245.777357] Lustre: fir-MDT0003: Client b0c2b6ec-d912-4 (at 10.50.0.62@o2ib2) reconnecting [2430245.785804] Lustre: Skipped 1 previous similar message [2430245.791146] Lustre: fir-MDT0003: Connection restored to (at 10.50.0.62@o2ib2) [2430245.798542] Lustre: Skipped 1 previous similar message [2430312.766653] Lustre: 66217:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (4/4), not sending early reply req@ffff8ed69e313f00 x1672365948243904/t0(0) o36->4e4cd82e-2195-4@10.50.4.72@o2ib2:554/0 lens 536/2888 e 17 to 0 dl 1596655679 ref 2 fl Interpret:/0/0 rc 0/0 [2430312.793992] Lustre: 66217:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 2 previous similar messages [2430318.142528] Lustre: fir-MDT0003: Client 4e4cd82e-2195-4 (at 10.50.4.72@o2ib2) reconnecting [2430318.150966] Lustre: Skipped 2 previous similar messages [2430318.156402] Lustre: fir-MDT0003: Connection restored to (at 10.50.4.72@o2ib2) [2430318.163804] Lustre: Skipped 2 previous similar messages [2430326.491051] LustreError: 67026:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1596655388, 300s ago), entering recovery for fir-MDT0000_UUID@10.0.10.51@o2ib7 ns: fir-MDT0000-osp-MDT0003 lock: ffff8ee526e9e300/0x375252c3eac5cce9 lrc: 4/0,1 mode: --/EX res: [0x200000004:0x1:0x0].0x0 bits 0x2/0x0 rrc: 27 type: IBT flags: 0x1000001000000 nid: local remote: 0x8a212c1ebb8fe1ee expref: -99 pid: 67026 timeout: 0 lvb_type: 0 [2430326.531300] LustreError: 67026:0:(ldlm_request.c:148:ldlm_expired_completion_wait()) Skipped 1 previous similar message [2430357.950965] LNet: Service thread pid 66160 was inactive for 537.68s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [2430357.968078] Pid: 66160, comm: mdt00_000 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2430357.978431] Call Trace: [2430357.981075] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [2430357.988222] [] ldlm_cli_enqueue_fini+0x96f/0xdf0 [ptlrpc] [2430357.995520] [] ldlm_cli_enqueue+0x40e/0x920 [ptlrpc] [2430358.002391] [] osp_md_object_lock+0x162/0x2d0 [osp] [2430358.009141] [] lod_object_lock+0xf4/0x780 [lod] [2430358.015539] [] mdd_object_lock+0x3e/0xe0 [mdd] [2430358.021870] [] mdt_remote_object_lock_try+0x1e1/0x750 [mdt] [2430358.029319] [] mdt_remote_object_lock+0x2a/0x30 [mdt] [2430358.036259] [] mdt_rename_lock+0xbe/0x4b0 [mdt] [2430358.042659] [] mdt_reint_rename+0x2c5/0x2b60 [mdt] [2430358.049321] [] mdt_reint_rec+0x83/0x210 [mdt] [2430358.055569] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2430358.062315] [] mdt_reint+0x67/0x140 [mdt] [2430358.068213] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2430358.075348] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2430358.083246] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2430358.089777] [] kthread+0xd1/0xe0 [2430358.094868] [] ret_from_fork_nospec_begin+0xe/0x21 [2430358.101537] [] 0xffffffffffffffff [2430358.106743] LustreError: dumping log to /tmp/lustre-log.1596655720.66160 [2430401.856255] Lustre: 66609:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff8ed8ced5a880 x1672514133968576/t0(0) o36->9eccbaf8-d387-4@10.49.0.64@o2ib1:644/0 lens 520/2888 e 3 to 0 dl 1596655769 ref 2 fl Interpret:/0/0 rc 0/0 [2430401.883505] Lustre: 66609:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 3 previous similar messages [2430408.555109] Lustre: fir-MDT0003: Client 9eccbaf8-d387-4 (at 10.49.0.64@o2ib1) reconnecting [2430408.563556] Lustre: Skipped 3 previous similar messages [2430444.455975] LNet: Service thread pid 67021 completed after 890.55s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [2430445.175381] LNet: Service thread pid 67018 completed after 872.98s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [2430445.191711] LNet: Skipped 1 previous similar message [2434893.877434] Lustre: fir-MDT0003: haven't heard from client 78dcdb4b-3df7-4 (at 10.50.7.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed7a72cbc00, cur 1596660256 expire 1596660106 last 1596660029 [2436199.195657] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.18@o2ib2) [2436199.203062] Lustre: Skipped 5 previous similar messages [2436264.424352] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.48@o2ib2) [2436340.401526] Lustre: fir-MDT0003: Connection restored to (at 10.50.7.43@o2ib2) [2436340.408929] Lustre: Skipped 1 previous similar message [2436782.339715] Lustre: fir-MDT0003: Connection restored to a5be94ff-265d-4 (at 10.50.5.57@o2ib2) [2436782.348419] Lustre: Skipped 3 previous similar messages [2441725.075015] Lustre: fir-MDT0003: haven't heard from client cf93f47a-fb6b-4 (at 10.50.17.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebf8636000, cur 1596667087 expire 1596666937 last 1596666860 [2441725.095169] Lustre: Skipped 6 previous similar messages [2441894.765335] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.8@o2ib2) [2442167.729136] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.40@o2ib2) [2442167.736634] Lustre: Skipped 1 previous similar message [2442214.089064] Lustre: fir-MDT0003: haven't heard from client b3df38e6-a065-4 (at 10.50.17.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ecb6bdf1c00, cur 1596667576 expire 1596667426 last 1596667349 [2442214.109121] Lustre: Skipped 2 previous similar messages [2442417.094955] Lustre: fir-MDT0003: haven't heard from client cf93f47a-fb6b-4 (at 10.50.17.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec4f0b3a800, cur 1596667779 expire 1596667629 last 1596667552 [2442752.325477] Lustre: fir-MDT0003: Connection restored to (at 10.50.17.8@o2ib2) [2442752.332884] Lustre: Skipped 1 previous similar message [2442995.111919] Lustre: fir-MDT0003: haven't heard from client b3df38e6-a065-4 (at 10.50.17.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed81ce98c00, cur 1596668357 expire 1596668207 last 1596668130 [2445400.901404] Lustre: fir-MDT0003: Client f9a49553-8328-4 (at 10.50.2.25@o2ib2) reconnecting [2445400.909847] Lustre: Skipped 1 previous similar message [2445400.915215] Lustre: fir-MDT0003: Connection restored to (at 10.50.2.25@o2ib2) [2488155.652040] LustreError: 66631:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2491487.139733] Lustre: fir-MDT0003: Client a5e00ba6-d534-4 (at 10.49.20.9@o2ib1) reconnecting [2491487.148199] Lustre: fir-MDT0003: Connection restored to (at 10.49.20.9@o2ib1) [2495151.014968] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [2495151.023465] Lustre: fir-MDT0003: Connection restored to (at 10.49.0.63@o2ib1) [2496149.135238] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [2496149.143711] Lustre: fir-MDT0003: Connection restored to (at 10.49.0.63@o2ib1) [2497641.080012] Lustre: fir-MDT0003: Client 3a0a94f3-822c-4 (at 10.49.13.25@o2ib1) reconnecting [2497641.088580] Lustre: fir-MDT0003: Connection restored to (at 10.49.13.25@o2ib1) [2515939.431636] LustreError: 66638:0:(mdt_open.c:1231:mdt_cross_open()) fir-MDT0003: [0x280039660:0xf678:0x0] doesn't exist!: rc = -14 [2515939.932231] LustreError: 66658:0:(mdt_open.c:1231:mdt_cross_open()) fir-MDT0003: [0x280039660:0xf73d:0x0] doesn't exist!: rc = -14 [2515939.944141] LustreError: 66658:0:(mdt_open.c:1231:mdt_cross_open()) Skipped 19 previous similar messages [2515941.326620] LustreError: 66805:0:(mdt_open.c:1231:mdt_cross_open()) fir-MDT0003: [0x280039660:0xf704:0x0] doesn't exist!: rc = -14 [2515941.338538] LustreError: 66805:0:(mdt_open.c:1231:mdt_cross_open()) Skipped 32 previous similar messages [2515943.326439] LustreError: 66784:0:(mdt_open.c:1231:mdt_cross_open()) fir-MDT0003: [0x280039660:0x1029c:0x0] doesn't exist!: rc = -14 [2515943.338456] LustreError: 66784:0:(mdt_open.c:1231:mdt_cross_open()) Skipped 192 previous similar messages [2515948.795769] LustreError: 66670:0:(mdt_open.c:1231:mdt_cross_open()) fir-MDT0003: [0x280039660:0xf864:0x0] doesn't exist!: rc = -14 [2515948.807691] LustreError: 66670:0:(mdt_open.c:1231:mdt_cross_open()) Skipped 248 previous similar messages [2515957.459699] LustreError: 66677:0:(mdt_open.c:1231:mdt_cross_open()) fir-MDT0003: [0x280039660:0xef29:0x0] doesn't exist!: rc = -14 [2515957.471610] LustreError: 66677:0:(mdt_open.c:1231:mdt_cross_open()) Skipped 48 previous similar messages [2525206.890122] Lustre: fir-MDT0003: Client b72056a9-84bd-4 (at 10.49.13.27@o2ib1) reconnecting [2525206.898828] Lustre: fir-MDT0003: Connection restored to (at 10.49.13.27@o2ib1) [2526144.481356] Lustre: fir-MDT0003: haven't heard from client fcf20583-b1c6-4 (at 10.50.5.4@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb5bac8800, cur 1596751504 expire 1596751354 last 1596751277 [2526637.494345] Lustre: fir-MDT0003: haven't heard from client b837e0e4-c6a2-4 (at 10.50.5.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efb5bacd000, cur 1596751997 expire 1596751847 last 1596751770 [2526637.514343] Lustre: Skipped 1 previous similar message [2526713.496252] Lustre: fir-MDT0003: haven't heard from client 113695d2-f132-4 (at 10.50.5.2@o2ib2) in 159 seconds. I think it's dead, and I am evicting it. exp ffff8efbf4f63400, cur 1596752073 expire 1596751923 last 1596751914 [2526847.488563] Lustre: fir-MDT0003: Connection restored to (at 10.50.5.1@o2ib2) [2527073.505775] Lustre: fir-MDT0003: haven't heard from client b5409223-ebe5-4 (at 10.50.5.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ee1a1f99c00, cur 1596752433 expire 1596752283 last 1596752206 [2527477.516463] Lustre: fir-MDT0003: Connection restored to (at 10.50.5.1@o2ib2) [2527713.522138] Lustre: fir-MDT0003: haven't heard from client b5409223-ebe5-4 (at 10.50.5.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8efa786f1c00, cur 1596753073 expire 1596752923 last 1596752846 [2537997.508289] Lustre: fir-MDT0003: Connection restored to 0b5da860-c582-4 (at 10.51.1.4@o2ib3) [2549569.744271] LustreError: 66686:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2550126.191845] LustreError: 66727:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2556455.749906] Lustre: fir-MDT0003: Client 0d073855-2aee-4 (at 10.50.4.47@o2ib2) reconnecting [2556455.758400] Lustre: fir-MDT0003: Connection restored to (at 10.50.4.47@o2ib2) [2563236.378875] Lustre: fir-MDT0003: Client d23b8c0b-9d6c-4 (at 10.50.2.9@o2ib2) reconnecting [2563236.387256] Lustre: fir-MDT0003: Connection restored to (at 10.50.2.9@o2ib2) [2595571.122860] Lustre: fir-MDT0003: Client 9c474a5d-4da7-4 (at 10.50.2.5@o2ib2) reconnecting [2595571.131244] Lustre: fir-MDT0003: Connection restored to (at 10.50.2.5@o2ib2) [2599415.982345] Lustre: fir-MDT0003: Connection restored to 0b5da860-c582-4 (at 10.51.1.4@o2ib3) [2607029.818154] LustreError: 66612:0:(mdd_dir.c:4496:mdd_migrate()) fir-MDD0003: '.ondemand' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H 2 .ondemand' to finish migration. [2612117.752521] Lustre: fir-MDT0003: Client f7adac06-daba-4 (at 10.49.26.5@o2ib1) reconnecting [2612117.761002] Lustre: fir-MDT0003: Connection restored to (at 10.49.26.5@o2ib1) [2619813.607928] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [2619813.616394] Lustre: fir-MDT0003: Connection restored to (at 10.49.0.63@o2ib1) [2623879.213277] Lustre: fir-MDT0003: haven't heard from client c92ac5da-c03b-4 (at 10.50.10.26@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8eebd6a40400, cur 1596849236 expire 1596849086 last 1596849009 [2636353.710151] Lustre: fir-MDT0003: Connection restored to a5be94ff-265d-4 (at 10.50.5.57@o2ib2) [2636384.564788] Lustre: fir-MDT0003: haven't heard from client a5be94ff-265d-4 (at 10.50.5.57@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ec4e0f42c00, cur 1596861741 expire 1596861591 last 1596861514 [2638270.405942] Lustre: fir-MDT0003: Connection restored to a5be94ff-265d-4 (at 10.50.5.57@o2ib2) [2638300.620695] Lustre: fir-MDT0003: haven't heard from client 8b101c27-e7f4-4 (at 10.50.5.57@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff8ed062701c00, cur 1596863657 expire 1596863507 last 1596863430 [2639627.935716] Lustre: fir-MDT0003: Client edb25609-39fb-4 (at 10.49.0.63@o2ib1) reconnecting [2639627.944184] Lustre: fir-MDT0003: Connection restored to (at 10.49.0.63@o2ib1) [2663131.800332] LustreError: 68703:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0xc90f:0x0]: rc = -2 [2663136.343377] LustreError: 100694:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x1697e:0x0]: rc = -2 [2663136.355987] LustreError: 100694:0:(mdd_object.c:3249:mdd_close()) Skipped 1 previous similar message [2663136.905828] LustreError: 66683:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x17604:0x0] not found: rc = -2 [2663137.608393] LustreError: 66810:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x1764d:0x0] not found: rc = -2 [2663137.620611] LustreError: 66810:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 3 previous similar messages [2663138.330280] LustreError: 126913:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x1771d:0x0]: rc = -2 [2663138.342891] LustreError: 126913:0:(mdd_object.c:3249:mdd_close()) Skipped 3 previous similar messages [2663138.705822] LustreError: 67056:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x17780:0x0] not found: rc = -2 [2663138.717962] LustreError: 67056:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 6 previous similar messages [2663140.353575] LustreError: 67074:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x17904:0x0]: rc = -2 [2663140.366103] LustreError: 67074:0:(mdd_object.c:3249:mdd_close()) Skipped 5 previous similar messages [2663140.950475] LustreError: 66692:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x17943:0x0] not found: rc = -2 [2663140.962567] LustreError: 66692:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 12 previous similar messages [2663144.507278] LustreError: 67061:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x17c2b:0x0]: rc = -2 [2663144.519822] LustreError: 67061:0:(mdd_object.c:3249:mdd_close()) Skipped 7 previous similar messages [2663145.078058] LustreError: 66755:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x17c57:0x0] not found: rc = -2 [2663145.090149] LustreError: 66755:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 18 previous similar messages [2663152.600233] LustreError: 66965:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x182ec:0x0]: rc = -2 [2663152.612774] LustreError: 66965:0:(mdd_object.c:3249:mdd_close()) Skipped 37 previous similar messages [2663153.134962] LustreError: 66753:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x1836f:0x0] not found: rc = -2 [2663153.147065] LustreError: 66753:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 38 previous similar messages [2663175.740735] LustreError: 67051:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x18bf2:0x0] not found: rc = -2 [2663175.752900] LustreError: 67051:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 63 previous similar messages [2663176.789503] LustreError: 66965:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x19344:0x0]: rc = -2 [2663176.802031] LustreError: 66965:0:(mdd_object.c:3249:mdd_close()) Skipped 50 previous similar messages [2663207.817889] LustreError: 66659:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x1aa2a:0x0] not found: rc = -2 [2663207.829992] LustreError: 66659:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 32 previous similar messages [2663209.047366] LustreError: 67065:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x1a8fc:0x0]: rc = -2 [2663209.059905] LustreError: 67065:0:(mdd_object.c:3249:mdd_close()) Skipped 22 previous similar messages [2663271.909174] LustreError: 66627:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044520:0x1db03:0x0] not found: rc = -2 [2663271.921256] LustreError: 66627:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 113 previous similar messages [2663273.161523] LustreError: 74686:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044520:0x1dbf1:0x0]: rc = -2 [2663273.174067] LustreError: 74686:0:(mdd_object.c:3249:mdd_close()) Skipped 94 previous similar messages [2663403.135529] LustreError: 66218:0:(mdd_object.c:400:mdd_xattr_get()) fir-MDD0003: object [0x280044522:0x4449:0x0] not found: rc = -2 [2663403.147555] LustreError: 66218:0:(mdd_object.c:400:mdd_xattr_get()) Skipped 378 previous similar messages [2663403.645800] LustreError: 100695:0:(mdd_object.c:3249:mdd_close()) fir-MDD0003: failed to get lu_attr of [0x280044522:0x4bea:0x0]: rc = -2 [2663403.658315] LustreError: 100695:0:(mdd_object.c:3249:mdd_close()) Skipped 323 previous similar messages [2663455.535626] LustreError: 66692:0:(lod_dev.c:132:lod_fld_lookup()) fir-MDT0003-mdtlov: invalid FID [0x0:0x0:0x0] [2663455.545914] LustreError: 66692:0:(lu_object.c:146:lu_object_put()) ASSERTION( top->loh_hash.next == ((void *)0) && top->loh_hash.pprev == ((void *)0) ) failed: [2663455.560424] LustreError: 66692:0:(lu_object.c:146:lu_object_put()) LBUG [2663455.567250] Pid: 66692, comm: mdt02_035 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [2663455.577613] Call Trace: [2663455.580267] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [2663455.587021] [] lbug_with_loc+0x4c/0xa0 [libcfs] [2663455.593439] [] lu_object_put+0x336/0x3e0 [obdclass] [2663455.600209] [] lu_object_put_nocache+0x16/0x20 [obdclass] [2663455.607502] [] lu_object_find_at+0x1fe/0xa60 [obdclass] [2663455.614604] [] lu_object_find+0x16/0x20 [obdclass] [2663455.621293] [] mdt_object_find+0x4b/0x170 [mdt] [2663455.627727] [] mdt_migrate_lookup.isra.40+0x158/0xa60 [mdt] [2663455.635193] [] mdt_reint_migrate+0x8bd/0x11d0 [mdt] [2663455.641939] [] mdt_reint_rec+0x83/0x210 [mdt] [2663455.648176] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2663455.654933] [] mdt_reint+0x67/0x140 [mdt] [2663455.660810] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663455.667959] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663455.675866] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663455.682391] [] kthread+0xd1/0xe0 [2663455.687479] [] ret_from_fork_nospec_begin+0xe/0x21 [2663455.694143] [] 0xffffffffffffffff [2663455.699346] Kernel panic - not syncing: LBUG [2663455.703792] CPU: 22 PID: 66692 Comm: mdt02_035 Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 [2663455.716670] Hardware name: Dell Inc. PowerEdge R6415/07YXFK, BIOS 1.12.2 11/15/2019 [2663455.724504] Call Trace: [2663455.727152] [] dump_stack+0x19/0x1b [2663455.732474] [] panic+0xe8/0x21f [2663455.737460] [] lbug_with_loc+0x9b/0xa0 [libcfs] [2663455.743849] [] lu_object_put+0x336/0x3e0 [obdclass] [2663455.750580] [] ? lu_object_start.isra.35+0x8b/0x120 [obdclass] [2663455.758261] [] lu_object_put_nocache+0x16/0x20 [obdclass] [2663455.765507] [] lu_object_find_at+0x1fe/0xa60 [obdclass] [2663455.772569] [] ? lod_xattr_get+0xee/0x700 [lod] [2663455.778947] [] lu_object_find+0x16/0x20 [obdclass] [2663455.785576] [] mdt_object_find+0x4b/0x170 [mdt] [2663455.791945] [] mdt_migrate_lookup.isra.40+0x158/0xa60 [mdt] [2663455.799349] [] mdt_reint_migrate+0x8bd/0x11d0 [mdt] [2663455.806077] [] ? check_unlink_entry+0x19/0xd0 [obdclass] [2663455.813234] [] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass] [2663455.820902] [] mdt_reint_rec+0x83/0x210 [mdt] [2663455.827089] [] mdt_reint_internal+0x6e3/0xaf0 [mdt] [2663455.833797] [] mdt_reint+0x67/0x140 [mdt] [2663455.839683] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [2663455.846777] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [2663455.854527] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [2663455.861785] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [2663455.869656] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [2663455.876621] [] ? __wake_up+0x44/0x50 [2663455.882061] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [2663455.888544] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [2663455.896114] [] kthread+0xd1/0xe0 [2663455.901175] [] ? insert_kthread_work+0x40/0x40 [2663455.907450] [] ret_from_fork_nospec_begin+0xe/0x21 [2663455.914070] [] ? insert_kthread_work+0x40/0x40