[    0.000000] Initializing cgroup subsys cpuset
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 3.10.0-957.5.1.el7.x86_64 (mockbuild@x86-019.build.eng.bos.redhat.com) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) ) #1 SMP Wed Dec 19 10:46:58 EST 2018
[    0.000000] Command line: initrd=initrd-3.10.0-957.5.1.el7.x86_64-gedi selinux=0 audit=0 panic=10 root=dhcp console=tty0 console=ttyS1,115200n8 console=ttyS2,115200n8 crashkernel=1G net.ifnames=0 biosdevname=0 pti=off scsi_mod.use_blk_mq=y dm_mod.use_blk_mq=y BOOT_IMAGE=vmlinuz-3.10.0-957.5.1.el7.x86_64 BOOTIF=01-24-6e-96-a4-96-18 
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009cfff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009d000-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x0000000060fd8fff] usable
[    0.000000] BIOS-e820: [mem 0x0000000060fd9000-0x0000000064fd8fff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000064fd9000-0x000000006c5cefff] usable
[    0.000000] BIOS-e820: [mem 0x000000006c5cf000-0x000000006e7cefff] reserved
[    0.000000] BIOS-e820: [mem 0x000000006e7cf000-0x000000006f5fefff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x000000006f5ff000-0x000000006f7fefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000006f7ff000-0x000000006f7fffff] usable
[    0.000000] BIOS-e820: [mem 0x000000006f800000-0x000000008fffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fd000000-0x00000000fe7fffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fec00000-0x00000000fec00fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fec80000-0x00000000fed00fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fed40000-0x00000000fed44fff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000ff000000-0x00000000ffffffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000603fffffff] usable
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 3.0 present.
[    0.000000] DMI: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x6040000 max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: uncachable
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-BFFFF uncachable
[    0.000000]   C0000-FFFFF write-protect
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 000000000000 mask 3FC000000000 write-back
[    0.000000]   1 base 004000000000 mask 3FE000000000 write-back
[    0.000000]   2 base 006000000000 mask 3FFFC0000000 write-back
[    0.000000]   3 base 000080000000 mask 3FFF80000000 uncachable
[    0.000000]   4 base 00007F000000 mask 3FFFFF000000 uncachable
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000]   8 disabled
[    0.000000]   9 disabled
[    0.000000] PAT configuration [0-7]: WB  WC  UC- UC  WB  WP  UC- UC  
[    0.000000] original variable MTRRs
[    0.000000] reg 0, base: 0GB, range: 256GB, type WB
[    0.000000] reg 1, base: 256GB, range: 128GB, type WB
[    0.000000] reg 2, base: 384GB, range: 1GB, type WB
[    0.000000] reg 3, base: 2GB, range: 2GB, type UC
[    0.000000] reg 4, base: 2032MB, range: 16MB, type UC
[    0.000000] total RAM covered: 392176M
[    0.000000]  gran_size: 64K 	chunk_size: 64K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 128K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 256K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 512K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 1M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 2M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 64K 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 64K 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 64K 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 64K 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 64K 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 64K 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 64K 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 64K 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 64K 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 128K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 256K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 512K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 1M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 2M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 128K 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 128K 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 128K 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 128K 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 128K 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 128K 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 128K 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 128K 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 128K 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 256K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 512K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 1M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 2M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 256K 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 256K 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 256K 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 256K 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 256K 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 256K 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 256K 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 256K 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 256K 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 512K 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 1M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 2M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 512K 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 512K 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 512K 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 512K 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 512K 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 512K 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 512K 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 512K 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 512K 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 1M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 2M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 1M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 1M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 1M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 1M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 1M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 1M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 1M 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 1M 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 1M 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 2M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 2M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 2M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 2M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 2M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 2M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 2M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 2M 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 2M 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 2M 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 4M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 4M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 4M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 4M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 4M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 4M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 4M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 4M 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 4M 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 4M 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 8M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 8M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 8M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 8M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 8M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 8M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 8M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 8M 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 8M 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 8M 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 16M 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 16M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 16M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 16M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 16M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 0G
[    0.000000]  gran_size: 16M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 0G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1G
[    0.000000]  gran_size: 16M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -56G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -96G
[    0.000000] *BAD*gran_size: 16M 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -64G
[    0.000000]  gran_size: 16M 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 16M 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 353G
[    0.000000]  gran_size: 32M 	chunk_size: 32M 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 16M
[    0.000000]  gran_size: 32M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 16M
[    0.000000]  gran_size: 32M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 16M
[    0.000000]  gran_size: 32M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 16M
[    0.000000]  gran_size: 32M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 16M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -1008M
[    0.000000]  gran_size: 32M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7152M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14320M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 32G 	num_reg: 10  	lose cover RAM: -28656M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 64G 	num_reg: 10  	lose cover RAM: -57328M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 128G 	num_reg: 10  	lose cover RAM: -98288M
[    0.000000] *BAD*gran_size: 32M 	chunk_size: 256G 	num_reg: 10  	lose cover RAM: -65520M
[    0.000000]  gran_size: 32M 	chunk_size: 512G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 1024G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 2048G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 4096G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 8192G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 16384G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 32M 	chunk_size: 32768G 	num_reg: 10  	lose cover RAM: 328720M
[    0.000000]  gran_size: 64M 	chunk_size: 64M 	num_reg: 10  	lose cover RAM: 263216M
[    0.000000]  gran_size: 64M 	chunk_size: 128M 	num_reg: 10  	lose cover RAM: 48M
[    0.000000]  gran_size: 64M 	chunk_size: 256M 	num_reg: 10  	lose cover RAM: 48M
[    0.000000]  gran_size: 64M 	chunk_size: 512M 	num_reg: 10  	lose cover RAM: 48M
[    0.000000]  gran_size: 64M 	chunk_size: 1G 	num_reg: 10  	lose cover RAM: 48M
[    0.000000] *BAD*gran_size: 64M 	chunk_size: 2G 	num_reg: 10  	lose cover RAM: -976M
[    0.000000]  gran_size: 64M 	chunk_size: 4G 	num_reg: 10  	lose cover RAM: 263216M
[    0.000000] *BAD*gran_size: 64M 	chunk_size: 8G 	num_reg: 10  	lose cover RAM: -7120M
[    0.000000] *BAD*gran_size: 64M 	chunk_size: 16G 	num_reg: 10  	lose cover RAM: -14288M
[    0.000000] mtrr_cleanup: can not find optimal value
[    0.000000] please specify mtrr_gran_size/mtrr_chunk_size
[    0.000000] e820: update [mem 0x7f000000-0xffffffff] usable ==> reserved
[    0.000000] e820: last_pfn = 0x6f800 max_arch_pfn = 0x400000000
[    0.000000] Base memory trampoline at [ffff9f8280097000] 97000 size 24576
[    0.000000] Using GB pages for direct mapping
[    0.000000] BRK [0x260c852000, 0x260c852fff] PGTABLE
[    0.000000] BRK [0x260c853000, 0x260c853fff] PGTABLE
[    0.000000] BRK [0x260c854000, 0x260c854fff] PGTABLE
[    0.000000] BRK [0x260c855000, 0x260c855fff] PGTABLE
[    0.000000] BRK [0x260c856000, 0x260c856fff] PGTABLE
[    0.000000] BRK [0x260c857000, 0x260c857fff] PGTABLE
[    0.000000] BRK [0x260c858000, 0x260c858fff] PGTABLE
[    0.000000] BRK [0x260c859000, 0x260c859fff] PGTABLE
[    0.000000] BRK [0x260c85a000, 0x260c85afff] PGTABLE
[    0.000000] RAMDISK: [mem 0x5f410000-0x60fb8fff]
[    0.000000] Early table checksum verification disabled
[    0.000000] ACPI: RSDP 00000000000fe320 00024 (v02 DELL  )
[    0.000000] ACPI: XSDT 000000006f6bf188 000F4 (v01 DELL   PE_SC3   00000000      01000013)
[    0.000000] ACPI: FACP 000000006f7f9000 00114 (v06 DELL   PE_SC3   00000000 DELL 00000001)
[    0.000000] ACPI: DSDT 000000006f79d000 33CDB (v02 DELL   PE_SC3   00000003 DELL 00000001)
[    0.000000] ACPI: FACS 000000006f50f000 00040
[    0.000000] ACPI: MCEJ 000000006f7fc000 00130 (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: WD__ 000000006f7fb000 00134 (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: SLIC 000000006f7fa000 00024 (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: HPET 000000006f7f8000 00038 (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: APIC 000000006f7f6000 016DE (v04 DELL   PE_SC3   00000000 DELL 00000001)
[    0.000000] ACPI: MCFG 000000006f7f5000 0003C (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: MIGT 000000006f7f4000 00040 (v01 DELL   PE_SC3   00000000 DELL 00000001)
[    0.000000] ACPI: MSCT 000000006f7f3000 00090 (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: PCAT 000000006f7d9000 00048 (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: PCCT 000000006f7d8000 0006E (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: RASF 000000006f7d7000 00030 (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: SLIT 000000006f7d6000 0006C (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: SRAT 000000006f7d3000 02830 (v03 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: SVOS 000000006f7d2000 00032 (v01 DELL   PE_SC3   00000000 DELL 00000001)
[    0.000000] ACPI: WSMT 000000006f7d1000 00028 (v01 DELL   PE_SC3   00000000 DELL 00000001)
[    0.000000] ACPI: OEM4 000000006f6fa000 A27C4 (v02  INTEL CPU  CST 00003000 INTL 20150818)
[    0.000000] ACPI: SSDT 000000006f6c4000 35130 (v02  INTEL SSDT  PM 00004000 INTL 20150818)
[    0.000000] ACPI: SSDT 000000006f6c3000 00761 (v02 DELL   PE_SC3   00000000 DELL 00000001)
[    0.000000] ACPI: SSDT 000000006f6c0000 02541 (v02  INTEL SpsNm    00000002 INTL 20150818)
[    0.000000] ACPI: SPCR 000000006f7fd000 00050 (v01                 00000000      00000000)
[    0.000000] ACPI: DMAR 000000006f7f2000 00118 (v01 DELL   PE_SC3   00000001 DELL 00000001)
[    0.000000] ACPI: HEST 000000006f7f1000 0017C (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: BERT 000000006f7f0000 00030 (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: ERST 000000006f7ef000 00230 (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: EINJ 000000006f7ee000 00150 (v01 DELL   PE_SC3   00000002 DELL 00000001)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x08 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x02 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x06 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x04 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x16 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x10 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x12 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x22 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x28 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x24 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x26 -> Node 0
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x100000000-0x603fffffff]
[    0.000000] NUMA: Initialized distance table, cnt=1
[    0.000000] NUMA: Node 0 [mem 0x00000000-0x7fffffff] + [mem 0x100000000-0x603fffffff] -> [mem 0x00000000-0x603fffffff]
[    0.000000] NODE_DATA(0) allocated [mem 0x603ffd8000-0x603fffefff]
[    0.000000] Reserving 1024MB of memory at 496MB for crashkernel (System RAM: 391813MB)
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x00001000-0x00ffffff]
[    0.000000]   DMA32    [mem 0x01000000-0xffffffff]
[    0.000000]   Normal   [mem 0x100000000-0x603fffffff]
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x00001000-0x0009cfff]
[    0.000000]   node   0: [mem 0x00100000-0x60fd8fff]
[    0.000000]   node   0: [mem 0x64fd9000-0x6c5cefff]
[    0.000000]   node   0: [mem 0x6f7ff000-0x6f7fffff]
[    0.000000]   node   0: [mem 0x100000000-0x603fffffff]
[    0.000000] Initmem setup node 0 [mem 0x00001000-0x603fffffff]
[    0.000000] On node 0 totalpages: 100304236
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3996 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 6616 pages used for memmap
[    0.000000]   DMA32 zone: 423376 pages, LIFO batch:31
[    0.000000]   Normal zone: 1560576 pages used for memmap
[    0.000000]   Normal zone: 99876864 pages, LIFO batch:31
[    0.000000] ACPI: PM-Timer IO Port: 0x508
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x04] lapic_id[0x08] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x02] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x06] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x04] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x07] lapic_id[0x16] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x05] lapic_id[0x10] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x06] lapic_id[0x12] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x08] lapic_id[0x22] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x28] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x09] lapic_id[0x24] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x26] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0xff] lapic_id[0xff] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x00] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x01] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x02] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x03] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x04] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x05] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x06] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x07] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x08] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x09] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x0a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x0b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x0c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x0d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x0e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x0f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x10] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x11] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x12] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x13] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x14] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x15] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x16] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x17] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x18] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x19] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x1a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x1b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x1c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x1d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x1e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x1f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x20] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x21] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x22] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x23] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x24] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x25] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x26] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x27] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x28] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x29] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x2a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x2b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x2c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x2d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x2e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x2f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x30] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x31] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x32] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x33] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x34] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x35] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x36] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x37] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x38] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x39] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x3a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x3b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x3c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x3d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x3e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x3f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x40] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x41] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x42] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x43] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x44] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x45] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x46] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x47] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x48] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x49] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x4a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x4b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x4c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x4d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x4e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x4f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x50] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x51] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x52] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x53] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x54] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x55] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x56] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x57] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x58] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x59] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x5a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x5b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x5c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x5d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x5e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x5f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x60] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x61] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x62] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x63] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x64] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x65] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x66] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x67] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x68] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x69] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x6a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x6b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x6c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x6d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x6e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x6f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x70] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x71] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x72] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x73] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x74] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x75] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x76] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x77] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x78] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x79] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x7a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x7b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x7c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x7d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x7e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x7f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x80] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x81] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x82] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x83] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x84] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x85] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x86] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x87] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x88] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x89] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x8a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x8b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x8c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x8d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x8e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x8f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x90] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x91] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x92] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x93] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x94] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x95] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x96] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x97] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x98] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x99] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x9a] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x9b] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x9c] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x9d] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x9e] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0x9f] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa0] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa1] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa2] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa3] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa4] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa5] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa6] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa7] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa8] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xa9] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xaa] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xab] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xac] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xad] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xae] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xaf] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb0] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb1] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb2] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb3] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb4] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb5] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb6] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb7] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb8] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xb9] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xba] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xbb] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xbc] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xbd] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xbe] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xbf] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc0] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc1] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc2] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc3] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc4] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc5] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc6] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc7] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc8] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xc9] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xca] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xcb] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xcc] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xcd] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xce] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xcf] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd0] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd1] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd2] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd3] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd4] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd5] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd6] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd7] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd8] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xd9] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xda] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xdb] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xdc] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xdd] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xde] disabled)
[    0.000000] ACPI: X2APIC (apic_id[0xffffffff] uid[0xdf] disabled)
[    0.000000] ACPI: X2APIC_NMI (uid[0xffffffff] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] high level lint[0x1])
[    0.000000] ACPI: IOAPIC (id[0x08] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: IOAPIC (id[0x09] address[0xfec01000] gsi_base[24])
[    0.000000] IOAPIC[1]: apic_id 9, version 32, address 0xfec01000, GSI 24-31
[    0.000000] ACPI: IOAPIC (id[0x0a] address[0xfec08000] gsi_base[32])
[    0.000000] IOAPIC[2]: apic_id 10, version 32, address 0xfec08000, GSI 32-39
[    0.000000] ACPI: IOAPIC (id[0x0b] address[0xfec10000] gsi_base[40])
[    0.000000] IOAPIC[3]: apic_id 11, version 32, address 0xfec10000, GSI 40-47
[    0.000000] ACPI: IOAPIC (id[0x0c] address[0xfec18000] gsi_base[48])
[    0.000000] IOAPIC[4]: apic_id 12, version 32, address 0xfec18000, GSI 48-55
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a701 base: 0xfed00000
[    0.000000] smpboot: Allowing 448 CPUs, 436 hotplug CPUs
[    0.000000] PM: Registered nosave memory: [mem 0x0009d000-0x0009ffff]
[    0.000000] PM: Registered nosave memory: [mem 0x000a0000-0x000dffff]
[    0.000000] PM: Registered nosave memory: [mem 0x000e0000-0x000fffff]
[    0.000000] PM: Registered nosave memory: [mem 0x60fd9000-0x64fd8fff]
[    0.000000] PM: Registered nosave memory: [mem 0x6c5cf000-0x6e7cefff]
[    0.000000] PM: Registered nosave memory: [mem 0x6e7cf000-0x6f5fefff]
[    0.000000] PM: Registered nosave memory: [mem 0x6f5ff000-0x6f7fefff]
[    0.000000] PM: Registered nosave memory: [mem 0x6f800000-0x8fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x90000000-0xfcffffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfd000000-0xfe7fffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfe800000-0xfebfffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfec00000-0xfec00fff]
[    0.000000] PM: Registered nosave memory: [mem 0xfec01000-0xfec7ffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfec80000-0xfed00fff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed01000-0xfed3ffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed40000-0xfed44fff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed45000-0xfeffffff]
[    0.000000] PM: Registered nosave memory: [mem 0xff000000-0xffffffff]
[    0.000000] e820: [mem 0x90000000-0xfcffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] setup_percpu: NR_CPUS:5120 nr_cpumask_bits:448 nr_cpu_ids:448 nr_node_ids:1
[    0.000000] PERCPU: Embedded 38 pages/cpu @ffff9fe139a00000 s118784 r8192 d28672 u262144
[    0.000000] pcpu-alloc: s118784 r8192 d28672 u262144 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 000 001 002 003 004 005 006 007 
[    0.000000] pcpu-alloc: [0] 008 009 010 011 012 013 014 015 
[    0.000000] pcpu-alloc: [0] 016 017 018 019 020 021 022 023 
[    0.000000] pcpu-alloc: [0] 024 025 026 027 028 029 030 031 
[    0.000000] pcpu-alloc: [0] 032 033 034 035 036 037 038 039 
[    0.000000] pcpu-alloc: [0] 040 041 042 043 044 045 046 047 
[    0.000000] pcpu-alloc: [0] 048 049 050 051 052 053 054 055 
[    0.000000] pcpu-alloc: [0] 056 057 058 059 060 061 062 063 
[    0.000000] pcpu-alloc: [0] 064 065 066 067 068 069 070 071 
[    0.000000] pcpu-alloc: [0] 072 073 074 075 076 077 078 079 
[    0.000000] pcpu-alloc: [0] 080 081 082 083 084 085 086 087 
[    0.000000] pcpu-alloc: [0] 088 089 090 091 092 093 094 095 
[    0.000000] pcpu-alloc: [0] 096 097 098 099 100 101 102 103 
[    0.000000] pcpu-alloc: [0] 104 105 106 107 108 109 110 111 
[    0.000000] pcpu-alloc: [0] 112 113 114 115 116 117 118 119 
[    0.000000] pcpu-alloc: [0] 120 121 122 123 124 125 126 127 
[    0.000000] pcpu-alloc: [0] 128 129 130 131 132 133 134 135 
[    0.000000] pcpu-alloc: [0] 136 137 138 139 140 141 142 143 
[    0.000000] pcpu-alloc: [0] 144 145 146 147 148 149 150 151 
[    0.000000] pcpu-alloc: [0] 152 153 154 155 156 157 158 159 
[    0.000000] pcpu-alloc: [0] 160 161 162 163 164 165 166 167 
[    0.000000] pcpu-alloc: [0] 168 169 170 171 172 173 174 175 
[    0.000000] pcpu-alloc: [0] 176 177 178 179 180 181 182 183 
[    0.000000] pcpu-alloc: [0] 184 185 186 187 188 189 190 191 
[    0.000000] pcpu-alloc: [0] 192 193 194 195 196 197 198 199 
[    0.000000] pcpu-alloc: [0] 200 201 202 203 204 205 206 207 
[    0.000000] pcpu-alloc: [0] 208 209 210 211 212 213 214 215 
[    0.000000] pcpu-alloc: [0] 216 217 218 219 220 221 222 223 
[    0.000000] pcpu-alloc: [0] 224 225 226 227 228 229 230 231 
[    0.000000] pcpu-alloc: [0] 232 233 234 235 236 237 238 239 
[    0.000000] pcpu-alloc: [0] 240 241 242 243 244 245 246 247 
[    0.000000] pcpu-alloc: [0] 248 249 250 251 252 253 254 255 
[    0.000000] pcpu-alloc: [0] 256 257 258 259 260 261 262 263 
[    0.000000] pcpu-alloc: [0] 264 265 266 267 268 269 270 271 
[    0.000000] pcpu-alloc: [0] 272 273 274 275 276 277 278 279 
[    0.000000] pcpu-alloc: [0] 280 281 282 283 284 285 286 287 
[    0.000000] pcpu-alloc: [0] 288 289 290 291 292 293 294 295 
[    0.000000] pcpu-alloc: [0] 296 297 298 299 300 301 302 303 
[    0.000000] pcpu-alloc: [0] 304 305 306 307 308 309 310 311 
[    0.000000] pcpu-alloc: [0] 312 313 314 315 316 317 318 319 
[    0.000000] pcpu-alloc: [0] 320 321 322 323 324 325 326 327 
[    0.000000] pcpu-alloc: [0] 328 329 330 331 332 333 334 335 
[    0.000000] pcpu-alloc: [0] 336 337 338 339 340 341 342 343 
[    0.000000] pcpu-alloc: [0] 344 345 346 347 348 349 350 351 
[    0.000000] pcpu-alloc: [0] 352 353 354 355 356 357 358 359 
[    0.000000] pcpu-alloc: [0] 360 361 362 363 364 365 366 367 
[    0.000000] pcpu-alloc: [0] 368 369 370 371 372 373 374 375 
[    0.000000] pcpu-alloc: [0] 376 377 378 379 380 381 382 383 
[    0.000000] pcpu-alloc: [0] 384 385 386 387 388 389 390 391 
[    0.000000] pcpu-alloc: [0] 392 393 394 395 396 397 398 399 
[    0.000000] pcpu-alloc: [0] 400 401 402 403 404 405 406 407 
[    0.000000] pcpu-alloc: [0] 408 409 410 411 412 413 414 415 
[    0.000000] pcpu-alloc: [0] 416 417 418 419 420 421 422 423 
[    0.000000] pcpu-alloc: [0] 424 425 426 427 428 429 430 431 
[    0.000000] pcpu-alloc: [0] 432 433 434 435 436 437 438 439 
[    0.000000] pcpu-alloc: [0] 440 441 442 443 444 445 446 447 
[    0.000000] Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 98736959
[    0.000000] Policy zone: Normal
[    0.000000] Kernel command line: initrd=initrd-3.10.0-957.5.1.el7.x86_64-gedi selinux=0 audit=0 panic=10 root=dhcp console=tty0 console=ttyS1,115200n8 console=ttyS2,115200n8 crashkernel=1G net.ifnames=0 biosdevname=0 pti=off scsi_mod.use_blk_mq=y dm_mod.use_blk_mq=y BOOT_IMAGE=vmlinuz-3.10.0-957.5.1.el7.x86_64 BOOTIF=01-24-6e-96-a4-96-18 
[    0.000000] audit: disabled (until reboot)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] x86/fpu: xstate_offset[2]: 0240, xstate_sizes[2]: 0100
[    0.000000] x86/fpu: xstate_offset[3]: 03c0, xstate_sizes[3]: 0040
[    0.000000] x86/fpu: xstate_offset[4]: 0400, xstate_sizes[4]: 0040
[    0.000000] x86/fpu: xstate_offset[5]: 0440, xstate_sizes[5]: 0040
[    0.000000] x86/fpu: xstate_offset[6]: 0480, xstate_sizes[6]: 0200
[    0.000000] x86/fpu: xstate_offset[7]: 0680, xstate_sizes[7]: 0400
[    0.000000] x86/fpu: xstate_offset[8]: 0000, xstate_sizes[8]: 0080
[    0.000000] x86/fpu: xstate_offset[9]: 0a80, xstate_sizes[9]: 0008
[    0.000000] xsave: enabled xstate_bv 0x2ff, cntxt size 0xa88 using standard form
[    0.000000] Memory: 3712664k/403701760k available (7664k kernel code, 2484816k absent, 7501548k reserved, 6055k data, 1876k init)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=448, Nodes=1
[    0.000000] x86/pti: Unmapping kernel while in userspace
[    0.000000] Hierarchical RCU implementation.
[    0.000000] 	RCU restricting CPUs from NR_CPUS=5120 to nr_cpu_ids=448.
[    0.000000] NR_IRQS:327936 nr_irqs:4552 0
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] console [ttyS1] enabled
[    0.000000] allocated 1605369856 bytes of page_cgroup
[    0.000000] please try 'cgroup_disable=memory' option if you don't want memory cgroups
[    0.000000] hpet clockevent registered
[    0.000000] tsc: Detected 3200.000 MHz processor
[    0.000083] Calibrating delay loop (skipped), value calculated using timer frequency.. 6400.00 BogoMIPS (lpj=3200000)
[    0.011102] pid_max: default: 458752 minimum: 3584
[    0.016377] Security Framework initialized
[    0.020676] SELinux:  Disabled at boot.
[    0.024718] Yama: becoming mindful.
[    0.048310] Dentry cache hash table entries: 67108864 (order: 17, 536870912 bytes)
[    0.110744] Inode-cache hash table entries: 33554432 (order: 16, 268435456 bytes)
[    0.141602] Mount-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[    0.149447] Mountpoint-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[    0.159308] Initializing cgroup subsys memory
[    0.163879] Initializing cgroup subsys devices
[    0.168521] Initializing cgroup subsys freezer
[    0.173164] Initializing cgroup subsys net_cls
[    0.177807] Initializing cgroup subsys blkio
[    0.185094] Initializing cgroup subsys perf_event
[    0.190013] Initializing cgroup subsys hugetlb
[    0.194657] Initializing cgroup subsys pids
[    0.199040] Initializing cgroup subsys net_prio
[    0.204873] mce: CPU supports 20 MCE banks
[    0.209184] CPU0: Thermal monitoring enabled (TM1)
[    0.214193] Last level iTLB entries: 4KB 0, 2MB 0, 4MB 0
[    0.219702] Last level dTLB entries: 4KB 64, 2MB 0, 4MB 0
[    0.225296] tlb_flushall_shift: 6
[    0.228835] Speculative Store Bypass: Mitigation: Speculative Store Bypass disabled via prctl and seccomp
[    0.238720] FEATURE SPEC_CTRL Present
[    0.242580] FEATURE IBPB_SUPPORT Present
[    0.246739] Spectre V2 : Mitigation: IBRS (kernel)
[    0.252170] Freeing SMP alternatives: 28k freed
[    0.258354] ACPI: Core revision 20130517
[    0.276226] ACPI: All ACPI Tables successfully acquired
[    0.294293] ftrace: allocating 29189 entries in 115 pages
[    0.309136] DMAR: Host address width 46
[    0.313172] DMAR: DRHD base: 0x000000c5ffc000 flags: 0x0
[    0.318682] DMAR: dmar0: reg_base_addr c5ffc000 ver 1:0 cap 8d2078c106f0466 ecap f020df
[    0.326987] DMAR: DRHD base: 0x000000e0ffc000 flags: 0x0
[    0.332497] DMAR: dmar1: reg_base_addr e0ffc000 ver 1:0 cap 8d2078c106f0466 ecap f020df
[    0.340802] DMAR: DRHD base: 0x000000fbffc000 flags: 0x0
[    0.346313] DMAR: dmar2: reg_base_addr fbffc000 ver 1:0 cap 8d2078c106f0466 ecap f020df
[    0.354618] DMAR: DRHD base: 0x000000aaffc000 flags: 0x1
[    0.360130] DMAR: dmar3: reg_base_addr aaffc000 ver 1:0 cap 8d2078c106f0466 ecap f020df
[    0.368436] DMAR: RMRR base: 0x0000006e2c9000 end: 0x0000006e7c8fff
[    0.374893] DMAR: RMRR base: 0x0000006f501000 end: 0x0000006f503fff
[    0.381352] DMAR: ATSR flags: 0x0
[    0.384872] DMAR-IR: IOAPIC id 12 under DRHD base  0xfbffc000 IOMMU 2
[    0.391503] DMAR-IR: IOAPIC id 11 under DRHD base  0xe0ffc000 IOMMU 1
[    0.398131] DMAR-IR: IOAPIC id 10 under DRHD base  0xc5ffc000 IOMMU 0
[    0.404764] DMAR-IR: IOAPIC id 8 under DRHD base  0xaaffc000 IOMMU 3
[    0.411310] DMAR-IR: IOAPIC id 9 under DRHD base  0xaaffc000 IOMMU 3
[    0.417852] DMAR-IR: HPET id 0 under DRHD base 0xaaffc000
[    0.423447] DMAR-IR: x2apic is disabled because BIOS sets x2apic opt out bit.
[    0.430483] DMAR-IR: Use 'intremap=no_x2apic_optout' to override the BIOS setting.
[    0.439217] DMAR-IR: Enabled IRQ remapping in xapic mode
[    0.444712] IRQ remapping doesn't support X2APIC mode, disable x2apic.
[    0.451439] Switched APIC routing to physical flat.
[    0.457457] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.473649] smpboot: CPU0: Intel(R) Xeon(R) Gold 6146 CPU @ 3.20GHz (fam: 06, model: 55, stepping: 04)
[    0.483544] TSC deadline timer enabled
[    0.483571] Performance Events: PEBS fmt3+, Skylake events, 32-deep LBR, full-width counters, Intel PMU driver.
[    0.494461] ... version:                4
[    0.498660] ... bit width:              48
[    0.502941] ... generic registers:      8
[    0.507132] ... value mask:             0000ffffffffffff
[    0.512633] ... max period:             00007fffffffffff
[    0.518130] ... fixed-purpose events:   3
[    0.522324] ... event mask:             00000007000000ff
[    0.543683] NMI watchdog: enabled on all CPUs, permanently consumes one hw-PMU counter.
[    0.531764] smpboot: Booting Node   0, Processors  #1 #2 #3 #4 #5 #6 #7 #8 #9 #10 #11
[    0.601919] Brought up 12 CPUs
[    0.605547] smpboot: Max logical packages: 38
[    0.610096] smpboot: Total of 12 processors activated (76800.00 BogoMIPS)
[    2.697829] node 0 initialised, 97500679 pages in 2026ms
[    2.703812] devtmpfs: initialized
[    2.745151] EVM: security.selinux
[    2.748660] EVM: security.ima
[    2.751824] EVM: security.capability
[    2.755734] PM: Registering ACPI NVS region [mem 0x6e7cf000-0x6f5fefff] (14876672 bytes)
[    2.765510] atomic64 test passed for x86-64 platform with CX8 and with SSE
[    2.772565] pinctrl core: initialized pinctrl subsystem
[    2.778100] RTC time: 11:32:22, date: 07/09/19
[    2.782915] NET: Registered protocol family 16
[    2.787809] ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
[    2.795674] ACPI: bus type PCI registered
[    2.799876] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    2.806708] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0x80000000-0x8fffffff] (base 0x80000000)
[    2.816303] PCI: MMCONFIG at [mem 0x80000000-0x8fffffff] reserved in E820
[    2.823274] pmd_set_huge: Cannot satisfy [mem 0x80000000-0x80200000] with a huge-page mapping due to MTRR override.
[    2.834420] PCI: Using configuration type 1 for base access
[    2.840183] PCI: Dell System detected, enabling pci=bfsort.
[    2.858604] ACPI: Added _OSI(Module Device)
[    2.862995] ACPI: Added _OSI(Processor Device)
[    2.867635] ACPI: Added _OSI(3.0 _SCP Extensions)
[    2.872536] ACPI: Added _OSI(Processor Aggregator Device)
[    2.878130] ACPI: Added _OSI(Linux-Dell-Video)
[    2.888033] ACPI: EC: Look up EC in DSDT
[    2.911614] ACPI: [Firmware Bug]: BIOS _OSI(Linux) query ignored
[    2.933445] ACPI: Dynamic OEM Table Load:
[    2.937842] ACPI: OEM4           (null) A27C4 (v02  INTEL CPU  CST 00003000 INTL 20150818)
[    2.969357] ACPI: Interpreter enabled
[    2.973221] ACPI: (supports S0 S5)
[    2.976810] ACPI: Using IOAPIC for interrupt routing
[    2.982001] HEST: Table parsing has been initialized.
[    2.987237] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    2.996701] ACPI: Enabled 5 GPEs in block 00 to 7F
[    3.033498] ACPI: PCI Root Bridge [PC00] (domain 0000 [bus 00-15])
[    3.039865] acpi PNP0A08:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI]
[    3.048365] acpi PNP0A08:00: PCIe AER handled by firmware
[    3.053976] acpi PNP0A08:00: _OSC: platform does not support [SHPCHotplug]
[    3.061091] acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug PME PCIeCapability]
[    3.069042] acpi PNP0A08:00: FADT indicates ASPM is unsupported, using BIOS configuration
[    3.077648] acpi PNP0A08:00: ignoring host bridge window [mem 0x000c4000-0x000cbfff window] (conflicts with Video ROM [mem 0x000c0000-0x000c7fff])
[    3.091837] PCI host bridge to bus 0000:00
[    3.096128] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    3.103096] pci_bus 0000:00: root bus resource [io  0x1000-0x3fff window]
[    3.110064] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    3.117846] pci_bus 0000:00: root bus resource [mem 0xfe010000-0xfe010fff window]
[    3.125627] pci_bus 0000:00: root bus resource [mem 0x90000000-0xaaffffff window]
[    3.133408] pci_bus 0000:00: root bus resource [mem 0x380000000000-0x380fffffffff window]
[    3.141882] pci_bus 0000:00: root bus resource [bus 00-15]
[    3.147577] pci 0000:00:00.0: [8086:2020] type 00 class 0x060000
[    3.147957] pci 0000:00:05.0: [8086:2024] type 00 class 0x088000
[    3.148311] pci 0000:00:05.2: [8086:2025] type 00 class 0x088000
[    3.148662] pci 0000:00:05.4: [8086:2026] type 00 class 0x080020
[    3.148668] pci 0000:00:05.4: reg 0x10: [mem 0x92d20000-0x92d20fff]
[    3.149021] pci 0000:00:08.0: [8086:2014] type 00 class 0x088000
[    3.149361] pci 0000:00:08.1: [8086:2015] type 00 class 0x110100
[    3.149697] pci 0000:00:08.2: [8086:2016] type 00 class 0x088000
[    3.150047] pci 0000:00:11.0: [8086:a1ec] type 00 class 0xff0000
[    3.150458] pci 0000:00:11.5: [8086:a1d2] type 00 class 0x010601
[    3.150470] pci 0000:00:11.5: reg 0x10: [mem 0x92d16000-0x92d17fff]
[    3.150476] pci 0000:00:11.5: reg 0x14: [mem 0x92d1f000-0x92d1f0ff]
[    3.150483] pci 0000:00:11.5: reg 0x18: [io  0x2068-0x206f]
[    3.150489] pci 0000:00:11.5: reg 0x1c: [io  0x2074-0x2077]
[    3.150496] pci 0000:00:11.5: reg 0x20: [io  0x2040-0x205f]
[    3.150502] pci 0000:00:11.5: reg 0x24: [mem 0x92c80000-0x92cfffff]
[    3.150535] pci 0000:00:11.5: PME# supported from D3hot
[    3.150892] pci 0000:00:14.0: [8086:a1af] type 00 class 0x0c0330
[    3.150907] pci 0000:00:14.0: reg 0x10: [mem 0x92d00000-0x92d0ffff 64bit]
[    3.150962] pci 0000:00:14.0: PME# supported from D3hot D3cold
[    3.151298] pci 0000:00:14.0: System wakeup disabled by ACPI
[    3.157163] pci 0000:00:14.2: [8086:a1b1] type 00 class 0x118000
[    3.157178] pci 0000:00:14.2: reg 0x10: [mem 0x92d1c000-0x92d1cfff 64bit]
[    3.157571] pci 0000:00:16.0: [8086:a1ba] type 00 class 0x078000
[    3.157592] pci 0000:00:16.0: reg 0x10: [mem 0x92d1b000-0x92d1bfff 64bit]
[    3.157667] pci 0000:00:16.0: PME# supported from D3hot
[    3.158007] pci 0000:00:16.1: [8086:a1bb] type 00 class 0x078000
[    3.158028] pci 0000:00:16.1: reg 0x10: [mem 0x92d1a000-0x92d1afff 64bit]
[    3.158103] pci 0000:00:16.1: PME# supported from D3hot
[    3.158446] pci 0000:00:16.4: [8086:a1be] type 00 class 0x078000
[    3.158467] pci 0000:00:16.4: reg 0x10: [mem 0x92d19000-0x92d19fff 64bit]
[    3.158543] pci 0000:00:16.4: PME# supported from D3hot
[    3.158883] pci 0000:00:17.0: [8086:a182] type 00 class 0x010601
[    3.158895] pci 0000:00:17.0: reg 0x10: [mem 0x92d14000-0x92d15fff]
[    3.158902] pci 0000:00:17.0: reg 0x14: [mem 0x92d1e000-0x92d1e0ff]
[    3.158908] pci 0000:00:17.0: reg 0x18: [io  0x2060-0x2067]
[    3.158914] pci 0000:00:17.0: reg 0x1c: [io  0x2070-0x2073]
[    3.158921] pci 0000:00:17.0: reg 0x20: [io  0x2020-0x203f]
[    3.158927] pci 0000:00:17.0: reg 0x24: [mem 0x92c00000-0x92c7ffff]
[    3.158959] pci 0000:00:17.0: PME# supported from D3hot
[    3.159314] pci 0000:00:1c.0: [8086:a190] type 01 class 0x060400
[    3.159364] pci 0000:00:1c.0: PME# supported from D0 D3hot D3cold
[    3.159711] pci 0000:00:1c.0: System wakeup disabled by ACPI
[    3.165579] pci 0000:00:1c.4: [8086:a194] type 01 class 0x060400
[    3.165628] pci 0000:00:1c.4: PME# supported from D0 D3hot D3cold
[    3.165976] pci 0000:00:1c.4: System wakeup disabled by ACPI
[    3.171851] pci 0000:00:1f.0: [8086:a1c1] type 00 class 0x060100
[    3.172286] pci 0000:00:1f.2: [8086:a1a1] type 00 class 0x058000
[    3.172296] pci 0000:00:1f.2: reg 0x10: [mem 0x92d10000-0x92d13fff]
[    3.172686] pci 0000:00:1f.4: [8086:a1a3] type 00 class 0x0c0500
[    3.172700] pci 0000:00:1f.4: reg 0x10: [mem 0x92d18000-0x92d180ff 64bit]
[    3.172721] pci 0000:00:1f.4: reg 0x20: [io  0x2000-0x201f]
[    3.173070] pci 0000:00:1f.5: [8086:a1a4] type 00 class 0x0c8000
[    3.173081] pci 0000:00:1f.5: reg 0x10: [mem 0xfe010000-0xfe010fff]
[    3.173495] pci 0000:00:1c.0: PCI bridge to [bus 01]
[    3.178654] pci 0000:00:1c.0:   bridge window [mem 0x92a00000-0x92bfffff]
[    3.178700] pci 0000:02:00.0: [1556:be00] type 01 class 0x060400
[    3.178806] pci 0000:02:00.0: System wakeup disabled by ACPI
[    3.186675] pci 0000:00:1c.4: PCI bridge to [bus 02-03]
[    3.192100] pci 0000:00:1c.4:   bridge window [mem 0x92000000-0x928fffff]
[    3.192107] pci 0000:00:1c.4:   bridge window [mem 0x91000000-0x91ffffff 64bit pref]
[    3.192171] pci 0000:03:00.0: [102b:0536] type 00 class 0x030000
[    3.192193] pci 0000:03:00.0: reg 0x10: [mem 0x91000000-0x91ffffff pref]
[    3.192207] pci 0000:03:00.0: reg 0x14: [mem 0x92808000-0x9280bfff]
[    3.192221] pci 0000:03:00.0: reg 0x18: [mem 0x92000000-0x927fffff]
[    3.192395] pci 0000:02:00.0: PCI bridge to [bus 03]
[    3.197558] pci 0000:02:00.0:   bridge window [mem 0x92000000-0x928fffff]
[    3.197565] pci 0000:02:00.0:   bridge window [mem 0x91000000-0x91ffffff 64bit pref]
[    3.197583] pci_bus 0000:00: on NUMA node 0
[    3.197846] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 6 10 *11 12 14 15)
[    3.206067] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 *6 10 11 12 14 15)
[    3.214224] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 *5 6 10 11 12 14 15)
[    3.222382] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 6 10 *11 12 14 15)
[    3.230544] ACPI: PCI Interrupt Link [LNKE] (IRQs 3 4 5 6 10 *11 12 14 15)
[    3.238714] ACPI: PCI Interrupt Link [LNKF] (IRQs 3 4 5 *6 10 11 12 14 15)
[    3.246884] ACPI: PCI Interrupt Link [LNKG] (IRQs 3 4 *5 6 10 11 12 14 15)
[    3.255035] ACPI: PCI Interrupt Link [LNKH] (IRQs 3 4 5 6 10 *11 12 14 15)
[    3.263405] ACPI: PCI Root Bridge [PC01] (domain 0000 [bus 16-63])
[    3.269769] acpi PNP0A08:01: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI]
[    3.278277] acpi PNP0A08:01: PCIe AER handled by firmware
[    3.283911] acpi PNP0A08:01: _OSC: platform does not support [SHPCHotplug]
[    3.291061] acpi PNP0A08:01: _OSC: OS now controls [PCIeHotplug PME PCIeCapability]
[    3.299015] acpi PNP0A08:01: FADT indicates ASPM is unsupported, using BIOS configuration
[    3.307639] PCI host bridge to bus 0000:16
[    3.311924] pci_bus 0000:16: root bus resource [io  0x4000-0x7fff window]
[    3.318891] pci_bus 0000:16: root bus resource [mem 0xab000000-0xc5ffffff window]
[    3.326673] pci_bus 0000:16: root bus resource [mem 0x381000000000-0x381fffffffff window]
[    3.335148] pci_bus 0000:16: root bus resource [bus 16-63]
[    3.340825] pci 0000:16:02.0: [8086:2032] type 01 class 0x060400
[    3.340892] pci 0000:16:02.0: PME# supported from D0 D3hot D3cold
[    3.340920] pci 0000:16:02.0: System wakeup disabled by ACPI
[    3.346789] pci 0000:16:05.0: [8086:2034] type 00 class 0x088000
[    3.346869] pci 0000:16:05.2: [8086:2035] type 00 class 0x088000
[    3.346946] pci 0000:16:05.4: [8086:2036] type 00 class 0x080020
[    3.346952] pci 0000:16:05.4: reg 0x10: [mem 0xab500000-0xab500fff]
[    3.347032] pci 0000:16:08.0: [8086:208d] type 00 class 0x088000
[    3.347095] pci 0000:16:08.1: [8086:208d] type 00 class 0x088000
[    3.347155] pci 0000:16:08.2: [8086:208d] type 00 class 0x088000
[    3.347214] pci 0000:16:08.3: [8086:208d] type 00 class 0x088000
[    3.347274] pci 0000:16:08.4: [8086:208d] type 00 class 0x088000
[    3.347333] pci 0000:16:08.5: [8086:208d] type 00 class 0x088000
[    3.347392] pci 0000:16:08.6: [8086:208d] type 00 class 0x088000
[    3.347451] pci 0000:16:08.7: [8086:208d] type 00 class 0x088000
[    3.347511] pci 0000:16:09.0: [8086:208d] type 00 class 0x088000
[    3.347572] pci 0000:16:09.1: [8086:208d] type 00 class 0x088000
[    3.347632] pci 0000:16:09.2: [8086:208d] type 00 class 0x088000
[    3.347691] pci 0000:16:09.3: [8086:208d] type 00 class 0x088000
[    3.347752] pci 0000:16:09.4: [8086:208d] type 00 class 0x088000
[    3.347812] pci 0000:16:09.5: [8086:208d] type 00 class 0x088000
[    3.347872] pci 0000:16:09.6: [8086:208d] type 00 class 0x088000
[    3.347932] pci 0000:16:09.7: [8086:208d] type 00 class 0x088000
[    3.347993] pci 0000:16:0a.0: [8086:208d] type 00 class 0x088000
[    3.348055] pci 0000:16:0a.1: [8086:208d] type 00 class 0x088000
[    3.348115] pci 0000:16:0a.2: [8086:208d] type 00 class 0x088000
[    3.348174] pci 0000:16:0a.3: [8086:208d] type 00 class 0x088000
[    3.348234] pci 0000:16:0a.4: [8086:208d] type 00 class 0x088000
[    3.348293] pci 0000:16:0a.5: [8086:208d] type 00 class 0x088000
[    3.348352] pci 0000:16:0a.6: [8086:208d] type 00 class 0x088000
[    3.348412] pci 0000:16:0a.7: [8086:208d] type 00 class 0x088000
[    3.348472] pci 0000:16:0b.0: [8086:208d] type 00 class 0x088000
[    3.348534] pci 0000:16:0b.1: [8086:208d] type 00 class 0x088000
[    3.348595] pci 0000:16:0b.2: [8086:208d] type 00 class 0x088000
[    3.348654] pci 0000:16:0b.3: [8086:208d] type 00 class 0x088000
[    3.348718] pci 0000:16:0e.0: [8086:208e] type 00 class 0x088000
[    3.348781] pci 0000:16:0e.1: [8086:208e] type 00 class 0x088000
[    3.348841] pci 0000:16:0e.2: [8086:208e] type 00 class 0x088000
[    3.348900] pci 0000:16:0e.3: [8086:208e] type 00 class 0x088000
[    3.348959] pci 0000:16:0e.4: [8086:208e] type 00 class 0x088000
[    3.349018] pci 0000:16:0e.5: [8086:208e] type 00 class 0x088000
[    3.349077] pci 0000:16:0e.6: [8086:208e] type 00 class 0x088000
[    3.349137] pci 0000:16:0e.7: [8086:208e] type 00 class 0x088000
[    3.349197] pci 0000:16:0f.0: [8086:208e] type 00 class 0x088000
[    3.349259] pci 0000:16:0f.1: [8086:208e] type 00 class 0x088000
[    3.349318] pci 0000:16:0f.2: [8086:208e] type 00 class 0x088000
[    3.349378] pci 0000:16:0f.3: [8086:208e] type 00 class 0x088000
[    3.349437] pci 0000:16:0f.4: [8086:208e] type 00 class 0x088000
[    3.349497] pci 0000:16:0f.5: [8086:208e] type 00 class 0x088000
[    3.349556] pci 0000:16:0f.6: [8086:208e] type 00 class 0x088000
[    3.349617] pci 0000:16:0f.7: [8086:208e] type 00 class 0x088000
[    3.349677] pci 0000:16:10.0: [8086:208e] type 00 class 0x088000
[    3.349742] pci 0000:16:10.1: [8086:208e] type 00 class 0x088000
[    3.349803] pci 0000:16:10.2: [8086:208e] type 00 class 0x088000
[    3.349863] pci 0000:16:10.3: [8086:208e] type 00 class 0x088000
[    3.349922] pci 0000:16:10.4: [8086:208e] type 00 class 0x088000
[    3.349982] pci 0000:16:10.5: [8086:208e] type 00 class 0x088000
[    3.350041] pci 0000:16:10.6: [8086:208e] type 00 class 0x088000
[    3.350101] pci 0000:16:10.7: [8086:208e] type 00 class 0x088000
[    3.350161] pci 0000:16:11.0: [8086:208e] type 00 class 0x088000
[    3.350224] pci 0000:16:11.1: [8086:208e] type 00 class 0x088000
[    3.350283] pci 0000:16:11.2: [8086:208e] type 00 class 0x088000
[    3.350344] pci 0000:16:11.3: [8086:208e] type 00 class 0x088000
[    3.350413] pci 0000:16:1d.0: [8086:2054] type 00 class 0x088000
[    3.350476] pci 0000:16:1d.1: [8086:2055] type 00 class 0x088000
[    3.350536] pci 0000:16:1d.2: [8086:2056] type 00 class 0x088000
[    3.350597] pci 0000:16:1d.3: [8086:2057] type 00 class 0x088000
[    3.350660] pci 0000:16:1e.0: [8086:2080] type 00 class 0x088000
[    3.350723] pci 0000:16:1e.1: [8086:2081] type 00 class 0x088000
[    3.350784] pci 0000:16:1e.2: [8086:2082] type 00 class 0x088000
[    3.350844] pci 0000:16:1e.3: [8086:2083] type 00 class 0x088000
[    3.350905] pci 0000:16:1e.4: [8086:2084] type 00 class 0x088000
[    3.350965] pci 0000:16:1e.5: [8086:2085] type 00 class 0x088000
[    3.351024] pci 0000:16:1e.6: [8086:2086] type 00 class 0x088000
[    3.351163] pci 0000:17:00.0: [8086:1521] type 00 class 0x020000
[    3.351178] pci 0000:17:00.0: reg 0x10: [mem 0xab300000-0xab3fffff]
[    3.351195] pci 0000:17:00.0: reg 0x1c: [mem 0xab40c000-0xab40ffff]
[    3.351212] pci 0000:17:00.0: reg 0x30: [mem 0xfff80000-0xffffffff pref]
[    3.351288] pci 0000:17:00.0: PME# supported from D0 D3hot D3cold
[    3.351313] pci 0000:17:00.0: reg 0x184: [mem 0x00000000-0x00003fff 64bit pref]
[    3.351314] pci 0000:17:00.0: VF(n) BAR0 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR0 for 8 VFs)
[    3.361786] pci 0000:17:00.0: reg 0x190: [mem 0x00000000-0x00003fff 64bit pref]
[    3.361788] pci 0000:17:00.0: VF(n) BAR3 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR3 for 8 VFs)
[    3.372358] pci 0000:17:00.1: [8086:1521] type 00 class 0x020000
[    3.372372] pci 0000:17:00.1: reg 0x10: [mem 0xab200000-0xab2fffff]
[    3.372389] pci 0000:17:00.1: reg 0x1c: [mem 0xab408000-0xab40bfff]
[    3.372405] pci 0000:17:00.1: reg 0x30: [mem 0xfff80000-0xffffffff pref]
[    3.372481] pci 0000:17:00.1: PME# supported from D0 D3hot D3cold
[    3.372502] pci 0000:17:00.1: reg 0x184: [mem 0x00000000-0x00003fff 64bit pref]
[    3.372503] pci 0000:17:00.1: VF(n) BAR0 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR0 for 8 VFs)
[    3.382978] pci 0000:17:00.1: reg 0x190: [mem 0x00000000-0x00003fff 64bit pref]
[    3.382979] pci 0000:17:00.1: VF(n) BAR3 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR3 for 8 VFs)
[    3.393539] pci 0000:17:00.2: [8086:1521] type 00 class 0x020000
[    3.393553] pci 0000:17:00.2: reg 0x10: [mem 0xab100000-0xab1fffff]
[    3.393570] pci 0000:17:00.2: reg 0x1c: [mem 0xab404000-0xab407fff]
[    3.393587] pci 0000:17:00.2: reg 0x30: [mem 0xfff80000-0xffffffff pref]
[    3.393662] pci 0000:17:00.2: PME# supported from D0 D3hot D3cold
[    3.393682] pci 0000:17:00.2: reg 0x184: [mem 0x00000000-0x00003fff 64bit pref]
[    3.393684] pci 0000:17:00.2: VF(n) BAR0 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR0 for 8 VFs)
[    3.404159] pci 0000:17:00.2: reg 0x190: [mem 0x00000000-0x00003fff 64bit pref]
[    3.404161] pci 0000:17:00.2: VF(n) BAR3 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR3 for 8 VFs)
[    3.414720] pci 0000:17:00.3: [8086:1521] type 00 class 0x020000
[    3.414734] pci 0000:17:00.3: reg 0x10: [mem 0xab000000-0xab0fffff]
[    3.414751] pci 0000:17:00.3: reg 0x1c: [mem 0xab400000-0xab403fff]
[    3.414768] pci 0000:17:00.3: reg 0x30: [mem 0xfff80000-0xffffffff pref]
[    3.414843] pci 0000:17:00.3: PME# supported from D0 D3hot D3cold
[    3.414864] pci 0000:17:00.3: reg 0x184: [mem 0x00000000-0x00003fff 64bit pref]
[    3.414865] pci 0000:17:00.3: VF(n) BAR0 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR0 for 8 VFs)
[    3.425340] pci 0000:17:00.3: reg 0x190: [mem 0x00000000-0x00003fff 64bit pref]
[    3.425341] pci 0000:17:00.3: VF(n) BAR3 space: [mem 0x00000000-0x0001ffff 64bit pref] (contains BAR3 for 8 VFs)
[    3.437832] pci 0000:16:02.0: PCI bridge to [bus 17]
[    3.442996] pci 0000:16:02.0:   bridge window [mem 0xab000000-0xab4fffff]
[    3.443011] pci_bus 0000:16: on NUMA node 0
[    3.443170] ACPI: PCI Root Bridge [PC02] (domain 0000 [bus 64-b1])
[    3.449541] acpi PNP0A08:02: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI]
[    3.458080] acpi PNP0A08:02: PCIe AER handled by firmware
[    3.463712] acpi PNP0A08:02: _OSC: platform does not support [SHPCHotplug]
[    3.470853] acpi PNP0A08:02: _OSC: OS now controls [PCIeHotplug PME PCIeCapability]
[    3.481672] acpi PNP0A08:02: FADT indicates ASPM is unsupported, using BIOS configuration
[    3.490249] PCI host bridge to bus 0000:64
[    3.494539] pci_bus 0000:64: root bus resource [io  0x8000-0xbfff window]
[    3.501505] pci_bus 0000:64: root bus resource [mem 0xc6000000-0xe0ffffff window]
[    3.509285] pci_bus 0000:64: root bus resource [mem 0x382000000000-0x382fffffffff window]
[    3.517760] pci_bus 0000:64: root bus resource [bus 64-b1]
[    3.523444] pci 0000:64:00.0: [8086:2030] type 01 class 0x060400
[    3.523509] pci 0000:64:00.0: PME# supported from D0 D3hot D3cold
[    3.523562] pci 0000:64:00.0: System wakeup disabled by ACPI
[    3.529428] pci 0000:64:05.0: [8086:2034] type 00 class 0x088000
[    3.529501] pci 0000:64:05.2: [8086:2035] type 00 class 0x088000
[    3.529574] pci 0000:64:05.4: [8086:2036] type 00 class 0x080020
[    3.529580] pci 0000:64:05.4: reg 0x10: [mem 0xc6200000-0xc6200fff]
[    3.529655] pci 0000:64:08.0: [8086:2066] type 00 class 0x088000
[    3.529725] pci 0000:64:09.0: [8086:2066] type 00 class 0x088000
[    3.529793] pci 0000:64:0a.0: [8086:2040] type 00 class 0x088000
[    3.529860] pci 0000:64:0a.1: [8086:2041] type 00 class 0x088000
[    3.529924] pci 0000:64:0a.2: [8086:2042] type 00 class 0x088000
[    3.529989] pci 0000:64:0a.3: [8086:2043] type 00 class 0x088000
[    3.530053] pci 0000:64:0a.4: [8086:2044] type 00 class 0x088000
[    3.530117] pci 0000:64:0a.5: [8086:2045] type 00 class 0x088000
[    3.530182] pci 0000:64:0a.6: [8086:2046] type 00 class 0x088000
[    3.530247] pci 0000:64:0a.7: [8086:2047] type 00 class 0x088000
[    3.530311] pci 0000:64:0b.0: [8086:2048] type 00 class 0x088000
[    3.530378] pci 0000:64:0b.1: [8086:2049] type 00 class 0x088000
[    3.530444] pci 0000:64:0b.2: [8086:204a] type 00 class 0x088000
[    3.530507] pci 0000:64:0b.3: [8086:204b] type 00 class 0x088000
[    3.530574] pci 0000:64:0c.0: [8086:2040] type 00 class 0x088000
[    3.530641] pci 0000:64:0c.1: [8086:2041] type 00 class 0x088000
[    3.530707] pci 0000:64:0c.2: [8086:2042] type 00 class 0x088000
[    3.530773] pci 0000:64:0c.3: [8086:2043] type 00 class 0x088000
[    3.530839] pci 0000:64:0c.4: [8086:2044] type 00 class 0x088000
[    3.530904] pci 0000:64:0c.5: [8086:2045] type 00 class 0x088000
[    3.530970] pci 0000:64:0c.6: [8086:2046] type 00 class 0x088000
[    3.531035] pci 0000:64:0c.7: [8086:2047] type 00 class 0x088000
[    3.531102] pci 0000:64:0d.0: [8086:2048] type 00 class 0x088000
[    3.531169] pci 0000:64:0d.1: [8086:2049] type 00 class 0x088000
[    3.531235] pci 0000:64:0d.2: [8086:204a] type 00 class 0x088000
[    3.531302] pci 0000:64:0d.3: [8086:204b] type 00 class 0x088000
[    3.531403] pci 0000:65:00.0: [1000:0097] type 00 class 0x010700
[    3.531413] pci 0000:65:00.0: reg 0x10: [io  0x8000-0x80ff]
[    3.531420] pci 0000:65:00.0: reg 0x14: [mem 0xc6100000-0xc610ffff 64bit]
[    3.531426] pci 0000:65:00.0: reg 0x1c: [mem 0xc6000000-0xc60fffff 64bit]
[    3.531434] pci 0000:65:00.0: reg 0x30: [mem 0xfff00000-0xffffffff pref]
[    3.531544] pci 0000:65:00.0: supports D1 D2
[    3.531564] pci 0000:65:00.0: reg 0x174: [mem 0x00000000-0x0000ffff 64bit]
[    3.531566] pci 0000:65:00.0: VF(n) BAR0 space: [mem 0x00000000-0x000fffff 64bit] (contains BAR0 for 16 VFs)
[    3.541696] pci 0000:65:00.0: reg 0x17c: [mem 0x00000000-0x000fffff 64bit]
[    3.541698] pci 0000:65:00.0: VF(n) BAR2 space: [mem 0x00000000-0x00ffffff 64bit] (contains BAR2 for 16 VFs)
[    3.551867] pci 0000:64:00.0: PCI bridge to [bus 65]
[    3.557020] pci 0000:64:00.0:   bridge window [io  0x8000-0x8fff]
[    3.557022] pci 0000:64:00.0:   bridge window [mem 0xc6000000-0xc61fffff]
[    3.557029] pci_bus 0000:64: on NUMA node 0
[    3.557081] ACPI: PCI Root Bridge [PC03] (domain 0000 [bus b2-ff])
[    3.563444] acpi PNP0A08:03: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI]
[    3.571953] acpi PNP0A08:03: PCIe AER handled by firmware
[    3.577580] acpi PNP0A08:03: _OSC: platform does not support [SHPCHotplug]
[    3.584714] acpi PNP0A08:03: _OSC: OS now controls [PCIeHotplug PME PCIeCapability]
[    3.592666] acpi PNP0A08:03: FADT indicates ASPM is unsupported, using BIOS configuration
[    3.601269] PCI host bridge to bus 0000:b2
[    3.605556] pci_bus 0000:b2: root bus resource [io  0xc000-0xffff window]
[    3.612524] pci_bus 0000:b2: root bus resource [mem 0xe1000000-0xfbffffff window]
[    3.620305] pci_bus 0000:b2: root bus resource [mem 0x383000000000-0x383fffffffff window]
[    3.628779] pci_bus 0000:b2: root bus resource [bus b2-ff]
[    3.634464] pci 0000:b2:00.0: [8086:2030] type 01 class 0x060400
[    3.634535] pci 0000:b2:00.0: PME# supported from D0 D3hot D3cold
[    3.634593] pci 0000:b2:00.0: System wakeup disabled by ACPI
[    3.640466] pci 0000:b2:05.0: [8086:2034] type 00 class 0x088000
[    3.640544] pci 0000:b2:05.2: [8086:2035] type 00 class 0x088000
[    3.640620] pci 0000:b2:05.4: [8086:2036] type 00 class 0x080020
[    3.640626] pci 0000:b2:05.4: reg 0x10: [mem 0xe4000000-0xe4000fff]
[    3.640706] pci 0000:b2:0e.0: [8086:2058] type 00 class 0x110100
[    3.640772] pci 0000:b2:0e.1: [8086:2059] type 00 class 0x088000
[    3.640837] pci 0000:b2:0f.0: [8086:2058] type 00 class 0x110100
[    3.640903] pci 0000:b2:0f.1: [8086:2059] type 00 class 0x088000
[    3.640967] pci 0000:b2:10.0: [8086:2058] type 00 class 0x110100
[    3.641033] pci 0000:b2:10.1: [8086:2059] type 00 class 0x088000
[    3.641099] pci 0000:b2:12.0: [8086:204c] type 00 class 0x110100
[    3.641163] pci 0000:b2:12.1: [8086:204d] type 00 class 0x110100
[    3.641219] pci 0000:b2:12.2: [8086:204e] type 00 class 0x088000
[    3.641276] pci 0000:b2:12.4: [8086:204c] type 00 class 0x110100
[    3.641339] pci 0000:b2:12.5: [8086:204d] type 00 class 0x110100
[    3.641397] pci 0000:b2:15.0: [8086:2018] type 00 class 0x088000
[    3.641458] pci 0000:b2:16.0: [8086:2018] type 00 class 0x088000
[    3.641518] pci 0000:b2:16.4: [8086:2018] type 00 class 0x088000
[    3.641575] pci 0000:b2:17.0: [8086:2018] type 00 class 0x088000
[    3.641727] pci 0000:b3:00.0: [15b3:1013] type 00 class 0x020700
[    3.641863] pci 0000:b3:00.0: reg 0x10: [mem 0xe2000000-0xe3ffffff 64bit pref]
[    3.642112] pci 0000:b3:00.0: reg 0x30: [mem 0xfff00000-0xffffffff pref]
[    3.642744] pci 0000:b3:00.0: PME# supported from D3cold
[    3.642951] pci 0000:b2:00.0: PCI bridge to [bus b3]
[    3.648114] pci 0000:b2:00.0:   bridge window [mem 0xe2000000-0xe3ffffff 64bit pref]
[    3.648119] pci_bus 0000:b2: on NUMA node 0
[    3.648539] vgaarb: device added: PCI:0000:03:00.0,decodes=io+mem,owns=io+mem,locks=none
[    3.656944] vgaarb: loaded
[    3.659850] vgaarb: bridge control possible 0000:03:00.0
[    3.665444] TECH PREVIEW: scsi-mq may not be fully supported.
Please review provided documentation for limitations.
[    3.677578] SCSI subsystem initialized
[    3.681533] ACPI: bus type USB registered
[    3.685738] usbcore: registered new interface driver usbfs
[    3.691411] usbcore: registered new interface driver hub
[    3.697072] usbcore: registered new device driver usb
[    3.702653] EDAC MC: Ver: 3.0.0
[    3.706189] PCI: Using ACPI for IRQ routing
[    3.713248] PCI: pci_cache_line_size set to 64 bytes
[    3.713400] Expanded resource reserved due to conflict with PNP0003:00
[    3.720113] e820: reserve RAM buffer [mem 0x0009d000-0x0009ffff]
[    3.720114] e820: reserve RAM buffer [mem 0x60fd9000-0x63ffffff]
[    3.720115] e820: reserve RAM buffer [mem 0x6c5cf000-0x6fffffff]
[    3.720116] e820: reserve RAM buffer [mem 0x6f800000-0x6fffffff]
[    3.720420] NetLabel: Initializing
[    3.724020] NetLabel:  domain hash size = 128
[    3.728565] NetLabel:  protocols = UNLABELED CIPSOv4
[    3.733731] NetLabel:  unlabeled traffic allowed by default
[    3.739735] hpet0: at MMIO 0xfed00000, IRQs 2, 8, 0, 0, 0, 0, 0, 0
[    3.746981] hpet0: 8 comparators, 64-bit 24.000000 MHz counter
[    3.755024] amd_nb: Cannot enumerate AMD northbridges
[    3.760336] Switched to clocksource hpet
[    3.770785] pnp: PnP ACPI init
[    3.774050] ACPI: bus type PNP registered
[    3.778668] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    3.778744] system 00:01: [io  0x0500-0x053f] could not be reserved
[    3.785205] system 00:01: [io  0x0400-0x047f] has been reserved
[    3.791317] system 00:01: [io  0x0540-0x057f] has been reserved
[    3.797429] system 00:01: [io  0x0600-0x061f] has been reserved
[    3.803542] system 00:01: [io  0x0ca0-0x0ca5] has been reserved
[    3.809656] system 00:01: [io  0x0880-0x0883] has been reserved
[    3.815768] system 00:01: [io  0x0800-0x081f] has been reserved
[    3.821880] system 00:01: [mem 0xfed1c000-0xfed3ffff] has been reserved
[    3.828684] system 00:01: [mem 0xfed45000-0xfed8bfff] has been reserved
[    3.835489] system 00:01: [mem 0xff000000-0xffffffff] has been reserved
[    3.842291] system 00:01: [mem 0xfee00000-0xfeefffff] has been reserved
[    3.849095] system 00:01: [mem 0xfed12000-0xfed1200f] has been reserved
[    3.855900] system 00:01: [mem 0xfed12010-0xfed1201f] has been reserved
[    3.862704] system 00:01: [mem 0xfed1b000-0xfed1bfff] has been reserved
[    3.869508] system 00:01: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.869694] pnp 00:02: Plug and Play ACPI device, IDs PNP0501 (active)
[    3.869859] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active)
[    3.869928] system 00:04: [mem 0xfd000000-0xfdabffff] has been reserved
[    3.876737] system 00:04: [mem 0xfdad0000-0xfdadffff] has been reserved
[    3.883541] system 00:04: [mem 0xfdb00000-0xfdffffff] has been reserved
[    3.890343] system 00:04: [mem 0xfe000000-0xfe00ffff] has been reserved
[    3.897148] system 00:04: [mem 0xfe011000-0xfe01ffff] has been reserved
[    3.903953] system 00:04: [mem 0xfe036000-0xfe03bfff] has been reserved
[    3.910756] system 00:04: [mem 0xfe03d000-0xfe3fffff] has been reserved
[    3.917561] system 00:04: [mem 0xfe410000-0xfe7fffff] has been reserved
[    3.924365] system 00:04: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.924497] system 00:05: [io  0x1000-0x10fe] has been reserved
[    3.930609] system 00:05: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.930969] pnp: PnP ACPI: found 6 devices
[    3.935268] ACPI: bus type PNP unregistered
[    3.946388] pci 0000:17:00.0: can't claim BAR 6 [mem 0xfff80000-0xffffffff pref]: no compatible bridge window
[    3.956605] pci 0000:17:00.1: can't claim BAR 6 [mem 0xfff80000-0xffffffff pref]: no compatible bridge window
[    3.966817] pci 0000:17:00.2: can't claim BAR 6 [mem 0xfff80000-0xffffffff pref]: no compatible bridge window
[    3.977025] pci 0000:17:00.3: can't claim BAR 6 [mem 0xfff80000-0xffffffff pref]: no compatible bridge window
[    3.987238] pci 0000:65:00.0: can't claim BAR 6 [mem 0xfff00000-0xffffffff pref]: no compatible bridge window
[    3.997447] pci 0000:b3:00.0: can't claim BAR 6 [mem 0xfff00000-0xffffffff pref]: no compatible bridge window
[    4.007664] pci 0000:00:1c.0: bridge window [io  0x1000-0x0fff] to [bus 01] add_size 1000
[    4.007666] pci 0000:00:1c.0: bridge window [mem 0x00100000-0x000fffff 64bit pref] to [bus 01] add_size 200000 add_align 100000
[    4.007681] pci 0000:00:1c.0: res[15]=[mem 0x00100000-0x000fffff 64bit pref] res_to_dev_res add_size 200000 min_align 100000
[    4.007683] pci 0000:00:1c.0: res[15]=[mem 0x00100000-0x002fffff 64bit pref] res_to_dev_res add_size 200000 min_align 100000
[    4.007684] pci 0000:00:1c.0: res[13]=[io  0x1000-0x0fff] res_to_dev_res add_size 1000 min_align 1000
[    4.007686] pci 0000:00:1c.0: res[13]=[io  0x1000-0x1fff] res_to_dev_res add_size 1000 min_align 1000
[    4.007690] pci 0000:00:1c.0: BAR 15: assigned [mem 0x380000000000-0x3800001fffff 64bit pref]
[    4.016521] pci 0000:00:1c.0: BAR 13: assigned [io  0x3000-0x3fff]
[    4.022893] pci 0000:00:1c.0: PCI bridge to [bus 01]
[    4.028052] pci 0000:00:1c.0:   bridge window [io  0x3000-0x3fff]
[    4.034341] pci 0000:00:1c.0:   bridge window [mem 0x92a00000-0x92bfffff]
[    4.041316] pci 0000:00:1c.0:   bridge window [mem 0x380000000000-0x3800001fffff 64bit pref]
[    4.050058] pci 0000:02:00.0: PCI bridge to [bus 03]
[    4.055220] pci 0000:02:00.0:   bridge window [mem 0x92000000-0x928fffff]
[    4.062197] pci 0000:02:00.0:   bridge window [mem 0x91000000-0x91ffffff 64bit pref]
[    4.070247] pci 0000:00:1c.4: PCI bridge to [bus 02-03]
[    4.075666] pci 0000:00:1c.4:   bridge window [mem 0x92000000-0x928fffff]
[    4.082643] pci 0000:00:1c.4:   bridge window [mem 0x91000000-0x91ffffff 64bit pref]
[    4.090693] pci_bus 0000:00: resource 4 [io  0x0000-0x0cf7 window]
[    4.090694] pci_bus 0000:00: resource 5 [io  0x1000-0x3fff window]
[    4.090695] pci_bus 0000:00: resource 6 [mem 0x000a0000-0x000bffff window]
[    4.090697] pci_bus 0000:00: resource 7 [mem 0xfe010000-0xfe010fff window]
[    4.090698] pci_bus 0000:00: resource 8 [mem 0x90000000-0xaaffffff window]
[    4.090699] pci_bus 0000:00: resource 9 [mem 0x380000000000-0x380fffffffff window]
[    4.090700] pci_bus 0000:01: resource 0 [io  0x3000-0x3fff]
[    4.090702] pci_bus 0000:01: resource 1 [mem 0x92a00000-0x92bfffff]
[    4.090703] pci_bus 0000:01: resource 2 [mem 0x380000000000-0x3800001fffff 64bit pref]
[    4.090704] pci_bus 0000:02: resource 1 [mem 0x92000000-0x928fffff]
[    4.090705] pci_bus 0000:02: resource 2 [mem 0x91000000-0x91ffffff 64bit pref]
[    4.090707] pci_bus 0000:03: resource 1 [mem 0x92000000-0x928fffff]
[    4.090708] pci_bus 0000:03: resource 2 [mem 0x91000000-0x91ffffff 64bit pref]
[    4.090713] pci 0000:16:02.0: bridge window [mem 0x00100000-0x000fffff 64bit pref] to [bus 17] add_size 100000 add_align 100000
[    4.090720] pci 0000:16:02.0: res[15]=[mem 0x00100000-0x000fffff 64bit pref] res_to_dev_res add_size 100000 min_align 100000
[    4.090722] pci 0000:16:02.0: res[15]=[mem 0x00100000-0x001fffff 64bit pref] res_to_dev_res add_size 100000 min_align 100000
[    4.090724] pci 0000:16:02.0: BAR 15: assigned [mem 0x381000000000-0x3810000fffff 64bit pref]
[    4.099554] pci 0000:17:00.0: res[7]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099555] pci 0000:17:00.0: res[10]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099557] pci 0000:17:00.1: res[7]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099558] pci 0000:17:00.1: res[10]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099560] pci 0000:17:00.2: res[7]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099561] pci 0000:17:00.2: res[10]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099562] pci 0000:17:00.3: res[7]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099564] pci 0000:17:00.3: res[10]=[mem 0x00000000-0xffffffffffffffff 64bit pref] res_to_dev_res add_size 20000 min_align 0
[    4.099566] pci 0000:17:00.0: BAR 6: assigned [mem 0xab480000-0xab4fffff pref]
[    4.107093] pci 0000:17:00.1: BAR 6: no space for [mem size 0x00080000 pref]
[    4.114329] pci 0000:17:00.1: BAR 6: failed to assign [mem size 0x00080000 pref]
[    4.122030] pci 0000:17:00.2: BAR 6: no space for [mem size 0x00080000 pref]
[    4.129267] pci 0000:17:00.2: BAR 6: failed to assign [mem size 0x00080000 pref]
[    4.136970] pci 0000:17:00.3: BAR 6: no space for [mem size 0x00080000 pref]
[    4.144206] pci 0000:17:00.3: BAR 6: failed to assign [mem size 0x00080000 pref]
[    4.151911] pci 0000:17:00.0: BAR 7: assigned [mem 0x381000000000-0x38100001ffff 64bit pref]
[    4.160655] pci 0000:17:00.0: BAR 10: assigned [mem 0x381000020000-0x38100003ffff 64bit pref]
[    4.169482] pci 0000:17:00.1: BAR 7: assigned [mem 0x381000040000-0x38100005ffff 64bit pref]
[    4.178233] pci 0000:17:00.1: BAR 10: assigned [mem 0x381000060000-0x38100007ffff 64bit pref]
[    4.187069] pci 0000:17:00.2: BAR 7: assigned [mem 0x381000080000-0x38100009ffff 64bit pref]
[    4.195819] pci 0000:17:00.2: BAR 10: assigned [mem 0x3810000a0000-0x3810000bffff 64bit pref]
[    4.204653] pci 0000:17:00.3: BAR 7: assigned [mem 0x3810000c0000-0x3810000dffff 64bit pref]
[    4.213394] pci 0000:17:00.3: BAR 10: assigned [mem 0x3810000e0000-0x3810000fffff 64bit pref]
[    4.222223] pci 0000:16:02.0: PCI bridge to [bus 17]
[    4.227387] pci 0000:16:02.0:   bridge window [mem 0xab000000-0xab4fffff]
[    4.234366] pci 0000:16:02.0:   bridge window [mem 0x381000000000-0x3810000fffff 64bit pref]
[    4.243108] pci_bus 0000:16: resource 4 [io  0x4000-0x7fff window]
[    4.243109] pci_bus 0000:16: resource 5 [mem 0xab000000-0xc5ffffff window]
[    4.243110] pci_bus 0000:16: resource 6 [mem 0x381000000000-0x381fffffffff window]
[    4.243112] pci_bus 0000:17: resource 1 [mem 0xab000000-0xab4fffff]
[    4.243113] pci_bus 0000:17: resource 2 [mem 0x381000000000-0x3810000fffff 64bit pref]
[    4.243120] pci 0000:65:00.0: BAR 6: no space for [mem size 0x00100000 pref]
[    4.250359] pci 0000:65:00.0: BAR 6: failed to assign [mem size 0x00100000 pref]
[    4.258062] pci 0000:65:00.0: BAR 9: no space for [mem size 0x01000000 64bit]
[    4.265383] pci 0000:65:00.0: BAR 9: failed to assign [mem size 0x01000000 64bit]
[    4.273175] pci 0000:65:00.0: BAR 7: no space for [mem size 0x00100000 64bit]
[    4.280498] pci 0000:65:00.0: BAR 7: failed to assign [mem size 0x00100000 64bit]
[    4.288286] pci 0000:64:00.0: PCI bridge to [bus 65]
[    4.293450] pci 0000:64:00.0:   bridge window [io  0x8000-0x8fff]
[    4.299735] pci 0000:64:00.0:   bridge window [mem 0xc6000000-0xc61fffff]
[    4.306714] pci_bus 0000:64: Some PCI device resources are unassigned, try booting with pci=realloc
[    4.316057] pci_bus 0000:64: resource 4 [io  0x8000-0xbfff window]
[    4.316058] pci_bus 0000:64: resource 5 [mem 0xc6000000-0xe0ffffff window]
[    4.316059] pci_bus 0000:64: resource 6 [mem 0x382000000000-0x382fffffffff window]
[    4.316060] pci_bus 0000:65: resource 0 [io  0x8000-0x8fff]
[    4.316062] pci_bus 0000:65: resource 1 [mem 0xc6000000-0xc61fffff]
[    4.316068] pci 0000:b2:00.0: BAR 14: assigned [mem 0xe1000000-0xe10fffff]
[    4.323138] pci 0000:b3:00.0: BAR 6: assigned [mem 0xe1000000-0xe10fffff pref]
[    4.330669] pci 0000:b2:00.0: PCI bridge to [bus b3]
[    4.335830] pci 0000:b2:00.0:   bridge window [mem 0xe1000000-0xe10fffff]
[    4.342807] pci 0000:b2:00.0:   bridge window [mem 0xe2000000-0xe3ffffff 64bit pref]
[    4.350857] pci_bus 0000:b2: resource 4 [io  0xc000-0xffff window]
[    4.350858] pci_bus 0000:b2: resource 5 [mem 0xe1000000-0xfbffffff window]
[    4.350859] pci_bus 0000:b2: resource 6 [mem 0x383000000000-0x383fffffffff window]
[    4.350861] pci_bus 0000:b3: resource 1 [mem 0xe1000000-0xe10fffff]
[    4.350862] pci_bus 0000:b3: resource 2 [mem 0xe2000000-0xe3ffffff 64bit pref]
[    4.350963] NET: Registered protocol family 2
[    4.356389] TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
[    4.364555] TCP bind hash table entries: 65536 (order: 8, 1048576 bytes)
[    4.371531] TCP: Hash tables configured (established 524288 bind 65536)
[    4.378356] TCP: reno registered
[    4.381908] UDP hash table entries: 65536 (order: 9, 2097152 bytes)
[    4.388624] UDP-Lite hash table entries: 65536 (order: 9, 2097152 bytes)
[    4.396179] NET: Registered protocol family 1
[    4.401287] pci 0000:03:00.0: Boot video device
[    4.401575] PCI: CLS 32 bytes, default 64
[    4.401611] Unpacking initramfs...
[    4.615321] Freeing initrd memory: 28324k freed
[    4.622597] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[    4.629228] software IO TLB [mem 0x685cf000-0x6c5cf000] (64MB) mapped at [ffff9f82e85cf000-ffff9f82ec5cefff]
[    4.639499] RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 655360 ms ovfl timer
[    4.650541] RAPL PMU: hw unit of domain pp0-core 2^-14 Joules
[    4.656478] RAPL PMU: hw unit of domain package 2^-14 Joules
[    4.662334] RAPL PMU: hw unit of domain dram 2^-16 Joules
[    4.674234] sha1_ssse3: Using AVX2 optimized SHA-1 implementation
[    4.680595] sha256_ssse3: Using AVX2 optimized SHA-256 implementation
[    4.690845] futex hash table entries: 131072 (order: 11, 8388608 bytes)
[    4.698292] Initialise system trusted keyring
[    4.721632] HugeTLB registered 1 GB page size, pre-allocated 0 pages
[    4.728175] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    4.735668] zpool: loaded
[    4.738491] zbud: loaded
[    4.741516] VFS: Disk quotas dquot_6.5.2
[    4.745760] Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    4.752678] msgmni has been set to 32768
[    4.756866] Key type big_key registered
[    4.761570] NET: Registered protocol family 38
[    4.766212] Key type asymmetric registered
[    4.770506] Asymmetric key parser 'x509' registered
[    4.775614] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 248)
[    4.783463] io scheduler noop registered
[    4.787583] io scheduler deadline registered (default)
[    4.792942] io scheduler cfq registered
[    4.796980] io scheduler mq-deadline registered
[    4.801707] io scheduler kyber registered
[    4.806237] pcieport 0000:00:1c.0: irq 25 for MSI/MSI-X
[    4.806366] pcieport 0000:00:1c.4: irq 26 for MSI/MSI-X
[    4.806542] pcieport 0000:16:02.0: irq 28 for MSI/MSI-X
[    4.806695] pcieport 0000:64:00.0: irq 30 for MSI/MSI-X
[    4.806767] pcieport 0000:b2:00.0: irq 31 for MSI/MSI-X
[    4.806817] pcieport 0000:00:1c.0: Signaling PME through PCIe PME interrupt
[    4.813970] pcie_pme 0000:00:1c.0:pcie001: service driver pcie_pme loaded
[    4.813979] pcieport 0000:00:1c.4: Signaling PME through PCIe PME interrupt
[    4.821126] pci 0000:02:00.0: Signaling PME through PCIe PME interrupt
[    4.827844] pci 0000:03:00.0: Signaling PME through PCIe PME interrupt
[    4.834561] pcie_pme 0000:00:1c.4:pcie001: service driver pcie_pme loaded
[    4.834568] pcieport 0000:16:02.0: Signaling PME through PCIe PME interrupt
[    4.841720] pci 0000:17:00.0: Signaling PME through PCIe PME interrupt
[    4.848445] pci 0000:17:00.1: Signaling PME through PCIe PME interrupt
[    4.855162] pci 0000:17:00.2: Signaling PME through PCIe PME interrupt
[    4.861880] pci 0000:17:00.3: Signaling PME through PCIe PME interrupt
[    4.868598] pcie_pme 0000:16:02.0:pcie001: service driver pcie_pme loaded
[    4.868606] pcieport 0000:64:00.0: Signaling PME through PCIe PME interrupt
[    4.875765] pci 0000:65:00.0: Signaling PME through PCIe PME interrupt
[    4.882484] pcie_pme 0000:64:00.0:pcie001: service driver pcie_pme loaded
[    4.882492] pcieport 0000:b2:00.0: Signaling PME through PCIe PME interrupt
[    4.889651] pci 0000:b3:00.0: Signaling PME through PCIe PME interrupt
[    4.896377] pcie_pme 0000:b2:00.0:pcie001: service driver pcie_pme loaded
[    4.896405] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[    4.902176] pciehp 0000:00:1c.0:pcie004: Slot #0 AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug+ Surprise+ Interlock- NoCompl+ LLActRep+
[    4.914825] pciehp 0000:00:1c.0:pcie004: service driver pciehp loaded
[    4.914828] pciehp: PCI Express Hot Plug Controller Driver version: 0.4
[    4.921641] shpchp 0000:02:00.0: Cannot get control of SHPC hotplug
[    4.928109] shpchp: Standard Hot Plug PCI Controller Driver version: 0.4
[    4.935052] intel_idle: MWAIT substates: 0x2020
[    4.935053] intel_idle: v0.4.1 model 0x55
[    4.935173] intel_idle: lapic_timer_reliable_states 0xffffffff
[    4.935308] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input0
[    4.943023] ACPI: Power Button [PWRF]
[    4.947177] ERST: Error Record Serialization Table (ERST) support is initialized.
[    4.954969] pstore: Registered erst as persistent store backend
[    4.961354] GHES: APEI firmware first mode is enabled by APEI bit and WHEA _OSC.
[    4.969125] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
[    4.996297] 00:02: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    5.022821] 00:03: ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A
[    5.028969] Non-volatile memory driver v1.3
[    5.033383] Linux agpgart interface v0.103
[    5.037801] crash memory driver: version 1.1
[    5.042353] rdac: device handler registered
[    5.046778] hp_sw: device handler registered
[    5.051259] emc: device handler registered
[    5.055699] alua: device handler registered
[    5.060099] libphy: Fixed MDIO Bus: probed
[    5.064427] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
[    5.071145] ehci-pci: EHCI PCI platform driver
[    5.075798] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
[    5.082167] ohci-pci: OHCI PCI platform driver
[    5.086819] uhci_hcd: USB Universal Host Controller Interface driver
[    5.093474] xhci_hcd 0000:00:14.0: xHCI Host Controller
[    5.098929] xhci_hcd 0000:00:14.0: new USB bus registered, assigned bus number 1
[    5.107705] xhci_hcd 0000:00:14.0: hcc params 0x200077c1 hci version 0x100 quirks 0x00009810
[    5.116450] xhci_hcd 0000:00:14.0: cache line size of 32 is not supported
[    5.116484] xhci_hcd 0000:00:14.0: irq 32 for MSI/MSI-X
[    5.116570] usb usb1: New USB device found, idVendor=1d6b, idProduct=0002
[    5.123553] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[    5.131083] usb usb1: Product: xHCI Host Controller
[    5.136159] usb usb1: Manufacturer: Linux 3.10.0-957.5.1.el7.x86_64 xhci-hcd
[    5.143393] usb usb1: SerialNumber: 0000:00:14.0
[    5.148293] hub 1-0:1.0: USB hub found
[    5.152251] hub 1-0:1.0: 16 ports detected
[    5.157507] xhci_hcd 0000:00:14.0: xHCI Host Controller
[    5.162984] xhci_hcd 0000:00:14.0: new USB bus registered, assigned bus number 2
[    5.170723] usb usb2: New USB device found, idVendor=1d6b, idProduct=0003
[    5.177700] usb usb2: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[    5.185229] usb usb2: Product: xHCI Host Controller
[    5.190304] usb usb2: Manufacturer: Linux 3.10.0-957.5.1.el7.x86_64 xhci-hcd
[    5.197541] usb usb2: SerialNumber: 0000:00:14.0
[    5.202438] hub 2-0:1.0: USB hub found
[    5.206398] hub 2-0:1.0: 10 ports detected
[    5.211032] usb: port power management may be unreliable
[    5.216845] usbcore: registered new interface driver usbserial_generic
[    5.223570] usbserial: USB Serial support registered for generic
[    5.229796] i8042: PNP: No PS/2 controller found. Probing ports directly.
[    6.302408] i8042: No controller found
[    6.306492] mousedev: PS/2 mouse device common for all mice
[    6.306501] tsc: Refined TSC clocksource calibration: 3192.499 MHz
[    6.318719] Switched to clocksource tsc
[    6.318878] rtc_cmos 00:00: RTC can wake from S4
[    6.319241] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[    6.319330] rtc_cmos 00:00: alarms up to one month, y3k, 114 bytes nvram, hpet irqs
[    6.319847] cpuidle: using governor menu
[    6.320493] hidraw: raw HID events driver (C) Jiri Kosina
[    6.320756] usbcore: registered new interface driver usbhid
[    6.320756] usbhid: USB HID core driver
[    6.320941] Detected 1 PCC Subspaces
[    6.320983] Registering PCC driver as Mailbox controller
[    6.321057] drop_monitor: Initializing network drop monitor service
[    6.321537] TCP: cubic registered
[    6.321546] Initializing XFRM netlink socket
[    6.321925] NET: Registered protocol family 10
[    6.323324] NET: Registered protocol family 17
[    6.323337] mpls_gso: MPLS GSO support
[    6.329432] intel_rdt: Intel RDT MB allocation detected
[    6.333995] microcode: sig=0x50654, pf=0x80, revision=0x2000050
[    6.334189] microcode: Microcode Update Driver: v2.01 <tigran@aivazian.fsnet.co.uk>, Peter Oruba
[    6.334351] PM: Hibernation image not present or could not be loaded.
[    6.334355] Loading compiled-in X.509 certificates
[    6.334920] Loaded X.509 cert 'Red Hat Enterprise Linux Driver Update Program (key 3): bf57f3e87362bc7229d9f465321773dfd1f77a80'
[    6.335465] Loaded X.509 cert 'Red Hat Enterprise Linux kpatch signing key: 4d38fd864ebe18c5f0b72e3852e2014c3a676fc8'
[    6.336009] Loaded X.509 cert 'Red Hat Enterprise Linux kernel signing key: cc41373a704120b2b606fc7366916dfc46a52e93'
[    6.336031] registered taskstats version 1
[    6.340053] Key type trusted registered
[    6.343522] Key type encrypted registered
[    6.343607] IMA: No TPM chip found, activating TPM-bypass! (rc=-19)
[    6.346909]   Magic number: 11:325:529
[    6.346911]   hash matches drivers/base/power/main.c:644
[    6.346929] event_source uncore_cha_17: hash matches
[    6.347244] memory memory2161: hash matches
[    6.347319] memory memory1325: hash matches
[    6.347389] memory memory538: hash matches
[    6.351664] rtc_cmos 00:00: setting system clock to 2019-07-09 11:32:25 UTC (1562671945)
[    6.513147] Freeing unused kernel memory: 1876k freed
[    6.518009] usb 1-14: new high-speed USB device number 2 using xhci_hcd
[    6.525352] Write protecting the kernel read-only data: 12288k
[    6.532574] Freeing unused kernel memory: 516k freed
[    6.538948] Freeing unused kernel memory: 600k freed
[    6.633178] dracut: dracut-033-554.el7
[    6.640741] usb 1-14: New USB device found, idVendor=1604, idProduct=10c0
[    6.647730] usb 1-14: New USB device strings: Mfr=0, Product=0, SerialNumber=0
[    6.655640] hub 1-14:1.0: USB hub found
[    6.659732] hub 1-14:1.0: 4 ports detected
[    6.814252] RPC: Registered named UNIX socket transport module.
[    6.820368] RPC: Registered udp transport module.
[    6.825286] RPC: Registered tcp transport module.
[    6.830186] RPC: Registered tcp NFSv4.1 backchannel transport module.
[    6.899979] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    6.907424] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    6.914772] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    6.922098] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    6.951342] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    6.958688] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    7.129711] pps_core: LinuxPPS API ver. 1 registered
[    7.134871] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
[    7.145606] PTP clock support registered
[    7.150812] dca service started, version 1.12.1
[    7.168031] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.4.0-k
[    7.175179] igb: Copyright (c) 2007-2014 Intel Corporation.
[    7.182113] igb 0000:17:00.0: irq 35 for MSI/MSI-X
[    7.182153] igb 0000:17:00.0: irq 35 for MSI/MSI-X
[    7.182169] igb 0000:17:00.0: irq 36 for MSI/MSI-X
[    7.182186] igb 0000:17:00.0: irq 37 for MSI/MSI-X
[    7.182202] igb 0000:17:00.0: irq 38 for MSI/MSI-X
[    7.182218] igb 0000:17:00.0: irq 39 for MSI/MSI-X
[    7.182234] igb 0000:17:00.0: irq 40 for MSI/MSI-X
[    7.182250] igb 0000:17:00.0: irq 41 for MSI/MSI-X
[    7.182266] igb 0000:17:00.0: irq 42 for MSI/MSI-X
[    7.182287] igb 0000:17:00.0: irq 43 for MSI/MSI-X
[    7.183067] mlx5_core 0000:b3:00.0: firmware version: 12.23.1020
[    7.231973] igb 0000:17:00.0: added PHC on eth0
[    7.236700] igb 0000:17:00.0: Intel(R) Gigabit Ethernet Network Connection
[    7.243766] igb 0000:17:00.0: eth0: (PCIe:5.0Gb/s:Width x4) 24:6e:96:a4:96:18
[    7.251375] igb 0000:17:00.0: eth0: PBA No: G10565-020
[    7.256706] igb 0000:17:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[    7.265121] igb 0000:17:00.1: irq 45 for MSI/MSI-X
[    7.265161] igb 0000:17:00.1: irq 45 for MSI/MSI-X
[    7.265182] igb 0000:17:00.1: irq 46 for MSI/MSI-X
[    7.265199] igb 0000:17:00.1: irq 47 for MSI/MSI-X
[    7.265215] igb 0000:17:00.1: irq 48 for MSI/MSI-X
[    7.265231] igb 0000:17:00.1: irq 49 for MSI/MSI-X
[    7.265247] igb 0000:17:00.1: irq 50 for MSI/MSI-X
[    7.265262] igb 0000:17:00.1: irq 51 for MSI/MSI-X
[    7.265278] igb 0000:17:00.1: irq 52 for MSI/MSI-X
[    7.265294] igb 0000:17:00.1: irq 53 for MSI/MSI-X
[    7.314417] igb 0000:17:00.1: added PHC on eth1
[    7.319147] igb 0000:17:00.1: Intel(R) Gigabit Ethernet Network Connection
[    7.329011] igb 0000:17:00.1: eth1: (PCIe:5.0Gb/s:Width x4) 24:6e:96:a4:96:19
[    7.336622] igb 0000:17:00.1: eth1: PBA No: G10565-020
[    7.341952] igb 0000:17:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[    7.349885] usb 1-14.1: new high-speed USB device number 3 using xhci_hcd
[    7.357231] igb 0000:17:00.2: irq 55 for MSI/MSI-X
[    7.357270] igb 0000:17:00.2: irq 55 for MSI/MSI-X
[    7.357287] igb 0000:17:00.2: irq 60 for MSI/MSI-X
[    7.357303] igb 0000:17:00.2: irq 61 for MSI/MSI-X
[    7.357320] igb 0000:17:00.2: irq 62 for MSI/MSI-X
[    7.357337] igb 0000:17:00.2: irq 63 for MSI/MSI-X
[    7.357354] igb 0000:17:00.2: irq 64 for MSI/MSI-X
[    7.357371] igb 0000:17:00.2: irq 65 for MSI/MSI-X
[    7.357387] igb 0000:17:00.2: irq 66 for MSI/MSI-X
[    7.357403] igb 0000:17:00.2: irq 67 for MSI/MSI-X
[    7.410123] igb 0000:17:00.2: added PHC on eth2
[    7.414847] igb 0000:17:00.2: Intel(R) Gigabit Ethernet Network Connection
[    7.421906] igb 0000:17:00.2: eth2: (PCIe:5.0Gb/s:Width x4) 24:6e:96:a4:96:1a
[    7.429509] igb 0000:17:00.2: eth2: PBA No: G10565-020
[    7.429797] usb 1-14.1: New USB device found, idVendor=1604, idProduct=10c0
[    7.429799] usb 1-14.1: New USB device strings: Mfr=0, Product=0, SerialNumber=0
[    7.430083] hub 1-14.1:1.0: USB hub found
[    7.430104] hub 1-14.1:1.0: 4 ports detected
[    7.458329] igb 0000:17:00.2: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[    7.466876] igb 0000:17:00.3: irq 69 for MSI/MSI-X
[    7.466916] igb 0000:17:00.3: irq 69 for MSI/MSI-X
[    7.466933] igb 0000:17:00.3: irq 70 for MSI/MSI-X
[    7.466949] igb 0000:17:00.3: irq 71 for MSI/MSI-X
[    7.466966] igb 0000:17:00.3: irq 72 for MSI/MSI-X
[    7.466982] igb 0000:17:00.3: irq 73 for MSI/MSI-X
[    7.466998] igb 0000:17:00.3: irq 74 for MSI/MSI-X
[    7.467016] igb 0000:17:00.3: irq 75 for MSI/MSI-X
[    7.467033] igb 0000:17:00.3: irq 76 for MSI/MSI-X
[    7.467049] igb 0000:17:00.3: irq 77 for MSI/MSI-X
[    7.492555] usb 1-14.4: new high-speed USB device number 4 using xhci_hcd
[    7.519947] igb 0000:17:00.3: added PHC on eth3
[    7.524683] igb 0000:17:00.3: Intel(R) Gigabit Ethernet Network Connection
[    7.531744] igb 0000:17:00.3: eth3: (PCIe:5.0Gb/s:Width x4) 24:6e:96:a4:96:1b
[    7.539350] igb 0000:17:00.3: eth3: PBA No: G10565-020
[    7.544675] igb 0000:17:00.3: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[    7.572662] usb 1-14.4: New USB device found, idVendor=1604, idProduct=10c0
[    7.579816] usb 1-14.4: New USB device strings: Mfr=0, Product=0, SerialNumber=0
[    7.588137] hub 1-14.4:1.0: USB hub found
[    7.592541] hub 1-14.4:1.0: 4 ports detected
[    8.200200] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[    9.713981] mlx5_core 0000:b3:00.0: irq 78 for MSI/MSI-X
[    9.714031] mlx5_core 0000:b3:00.0: irq 79 for MSI/MSI-X
[    9.714078] mlx5_core 0000:b3:00.0: irq 80 for MSI/MSI-X
[    9.714125] mlx5_core 0000:b3:00.0: irq 81 for MSI/MSI-X
[    9.714170] mlx5_core 0000:b3:00.0: irq 82 for MSI/MSI-X
[    9.714218] mlx5_core 0000:b3:00.0: irq 83 for MSI/MSI-X
[    9.714263] mlx5_core 0000:b3:00.0: irq 84 for MSI/MSI-X
[    9.714308] mlx5_core 0000:b3:00.0: irq 85 for MSI/MSI-X
[    9.714367] mlx5_core 0000:b3:00.0: irq 86 for MSI/MSI-X
[    9.714412] mlx5_core 0000:b3:00.0: irq 87 for MSI/MSI-X
[    9.714458] mlx5_core 0000:b3:00.0: irq 88 for MSI/MSI-X
[    9.714502] mlx5_core 0000:b3:00.0: irq 89 for MSI/MSI-X
[    9.714547] mlx5_core 0000:b3:00.0: irq 90 for MSI/MSI-X
[    9.714592] mlx5_core 0000:b3:00.0: irq 91 for MSI/MSI-X
[    9.714634] mlx5_core 0000:b3:00.0: irq 92 for MSI/MSI-X
[    9.714677] mlx5_core 0000:b3:00.0: irq 93 for MSI/MSI-X
[    9.736027] mlx5_core 0000:b3:00.0: Port module event: module 0, Cable plugged
[   11.574347] igb 0000:17:00.0 eth0: igb: eth0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX/TX
[   11.584146] IPv6: ADDRCONF(NETDEV_CHANGE): eth0: link becomes ready
[   11.649461] dracut: Starting dhcp for interface eth0
[   13.806143] igb 0000:17:00.0: changing MTU from 1500 to 9000
[   17.729892] igb 0000:17:00.0 eth0: igb: eth0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX/TX
[   18.537496] FS-Cache: Loaded
[   18.561001] FS-Cache: Netfs 'nfs' registered for caching
[   18.572456] Key type dns_resolver registered
[   18.598649] NFS: Registering the id_resolver key type
[   18.603897] Key type id_resolver registered
[   18.608277] Key type id_legacy registered
[   18.671406] random: fast init done
[   18.731741] dracut: Mounted root filesystem 172.25.17.2:/images/nfsroot/f_production.20190627
[   18.951846] dracut: Switching root
[   19.094669] ip_tables: (C) 2000-2006 Netfilter Core Team
[   19.100366] systemd[1]: Inserted module 'ip_tables'
[   19.111153] systemd[1]: systemd 219 running in system mode. (+PAM +AUDIT +SELINUX +IMA -APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 -SECCOMP +BLKID +ELFUTILS +KMOD +IDN)
[   19.129830] systemd[1]: Detected architecture x86-64.
[   19.663769] random: crng init done
[   20.194095] ACPI Error: No handler for Region [SYSI] (ffff9f83ae8873f0) [IPMI] (20130517/evregion-162)
[   20.201446] ACPI Error: 
[   20.201447] Region IPMI (ID=7) has no handler
[   20.201449]  (20130517/exfldio-305)
[   20.201451] ACPI Error: 
[   20.201453] Method parse/execution failed 
[   20.201454] [\_SB_.PMI0._GHL] (Node ffff9fa88c543550)
[   20.201455] , AE_NOT_EXIST
[   20.201457]  (20130517/psparse-536)
[   20.201462] ACPI Error: 
[   20.201463] Method parse/execution failed 
[   20.201464] [\_SB_.PMI0._PMC] (Node ffff9fa88c5434b0)
[   20.201465] , AE_NOT_EXIST
[   20.201466]  (20130517/psparse-536)
[   20.201471] ACPI Exception: AE_NOT_EXIST, 
[   20.201472] Evaluating _PMC
[   20.201473]  (20130517/power_meter-753)
[   20.441672] ipmi message handler version 39.2
[   20.448740] ipmi device interface
[   20.460564] IPMI System Interface driver.
[   20.464783] ipmi_si ipmi_si.0: ipmi_platform: probing via SMBIOS
[   20.470981] ipmi_si: SMBIOS: io 0xca8 regsize 1 spacing 4 irq 10
[   20.477219] ipmi_si: Adding SMBIOS-specified kcs state machine
[   20.483252] ipmi_si IPI0001:00: ipmi_platform: probing via ACPI
[   20.489715] ipmi_si IPI0001:00: [io  0x0ca8] regsize 1 spacing 4 irq 10
[   20.489856] wmi_bus wmi_bus-PNP0C14:00: WQBC data block query control method not found
[   20.505139] ipmi_si ipmi_si.0: Removing SMBIOS-specified kcs state machine in favor of ACPI
[   20.513791] ipmi_si: Adding ACPI-specified kcs state machine
[   20.520152] ipmi_si: Trying ACPI-specified kcs state machine at i/o address 0xca8, slave address 0x20, irq 10
[   20.530517] lpc_ich 0000:00:1f.0: I/O space for ACPI uninitialized
[   20.536884] lpc_ich 0000:00:1f.0: No MFD cells added
[   20.544728] libata version 3.00 loaded.
[   20.545042] i801_smbus 0000:00:1f.4: SMBus using PCI interrupt
[   20.553050] mei_me 0000:00:16.0: irq 94 for MSI/MSI-X
[   20.560743] ahci 0000:00:11.5: version 3.0
[   20.560896] ahci 0000:00:11.5: irq 95 for MSI/MSI-X
[   20.560938] ahci 0000:00:11.5: AHCI 0001.0301 32 slots 6 ports 6 Gbps 0x3f impl SATA mode
[   20.569421] ahci 0000:00:11.5: flags: 64bit ncq sntf pm led clo only pio slum part ems deso sadm sds apst 
[   20.570891] ipmi_si IPI0001:00: The BMC does not support setting the recv irq bit, compensating, but the BMC needs to be fixed.
[   20.573924] ipmi_si IPI0001:00: Using irq 10
[   20.576115] ipmi_si IPI0001:00: Found new BMC (man_id: 0x0002a2, prod_id: 0x0100, dev_id: 0x20)
[   20.615891] scsi host0: ahci
[   20.619209] scsi host1: ahci
[   20.636598] scsi host2: ahci
[   20.638001] ipmi_si IPI0001:00: IPMI kcs interface initialized
[   20.646416] scsi host3: ahci
[   20.650220] scsi host4: ahci
[   20.666867] scsi host5: ahci
[   20.670080] ata1: SATA max UDMA/133 abar m524288@0x92c80000 port 0x92c80100 irq 95
[   20.680748] ata2: SATA max UDMA/133 abar m524288@0x92c80000 port 0x92c80180 irq 95
[   20.688616] ata3: SATA max UDMA/133 abar m524288@0x92c80000 port 0x92c80200 irq 95
[   20.696495] ata4: SATA max UDMA/133 abar m524288@0x92c80000 port 0x92c80280 irq 95
[   20.704371] ata5: SATA max UDMA/133 abar m524288@0x92c80000 port 0x92c80300 irq 95
[   20.712243] ata6: SATA max UDMA/133 abar m524288@0x92c80000 port 0x92c80380 irq 95
[   20.724084] ahci 0000:00:17.0: irq 96 for MSI/MSI-X
[   20.734622] ahci 0000:00:17.0: AHCI 0001.0301 32 slots 8 ports 6 Gbps 0xff impl SATA mode
[   20.743102] ahci 0000:00:17.0: flags: 64bit ncq sntf pm led clo only pio slum part ems deso sadm sds apst 
[   20.774917] scsi host6: ahci
[   20.780289] scsi host7: ahci
[   20.781214] [TTM] Zone  kernel: Available graphics memory: 196873370 kiB
[   20.781215] [TTM] Zone   dma32: Available graphics memory: 2097152 kiB
[   20.781215] [TTM] Initializing pool allocator
[   20.781217] [TTM] Initializing DMA pool allocator
[   20.808130] scsi host8: ahci
[   20.812112] scsi host9: ahci
[   20.816346] fbcon: mgadrmfb (fb0) is primary device
[   20.819954] scsi host10: ahci
[   20.821920] scsi host11: ahci
[   20.832221] scsi host12: ahci
[   20.835806] scsi host13: ahci
[   20.835909] ata7: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00100 irq 96
[   20.835911] ata8: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00180 irq 96
[   20.835912] ata9: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00200 irq 96
[   20.835914] ata10: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00280 irq 96
[   20.835915] ata11: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00300 irq 96
[   20.835917] ata12: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00380 irq 96
[   20.835918] ata13: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00400 irq 96
[   20.835919] ata14: SATA max UDMA/133 abar m524288@0x92c00000 port 0x92c00480 irq 96
[   20.997528] mpt3sas version 16.100.01.00 loaded
[   20.999711] mpt3sas_cm0: 64 BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (393746740 kB)
[   21.013610] input: PC Speaker as /devices/platform/pcspkr/input/input1
[   21.023769] ata5: SATA link down (SStatus 0 SControl 300)
[   21.023785] ata2: SATA link down (SStatus 0 SControl 300)
[   21.023800] ata4: SATA link down (SStatus 0 SControl 300)
[   21.023816] ata3: SATA link down (SStatus 0 SControl 300)
[   21.023949] ata6: SATA link down (SStatus 0 SControl 300)
[   21.024760] ata1: SATA link down (SStatus 0 SControl 300)
[   21.057328] mpt3sas_cm0: MSI-X vectors supported: 96, no of cores: 12, max_msix_vectors: -1
[   21.057432] mpt3sas 0000:65:00.0: irq 98 for MSI/MSI-X
[   21.057449] mpt3sas 0000:65:00.0: irq 99 for MSI/MSI-X
[   21.057465] mpt3sas 0000:65:00.0: irq 100 for MSI/MSI-X
[   21.057480] mpt3sas 0000:65:00.0: irq 101 for MSI/MSI-X
[   21.057495] mpt3sas 0000:65:00.0: irq 102 for MSI/MSI-X
[   21.057510] mpt3sas 0000:65:00.0: irq 103 for MSI/MSI-X
[   21.057525] mpt3sas 0000:65:00.0: irq 104 for MSI/MSI-X
[   21.057540] mpt3sas 0000:65:00.0: irq 105 for MSI/MSI-X
[   21.057555] mpt3sas 0000:65:00.0: irq 106 for MSI/MSI-X
[   21.057570] mpt3sas 0000:65:00.0: irq 107 for MSI/MSI-X
[   21.057586] mpt3sas 0000:65:00.0: irq 108 for MSI/MSI-X
[   21.057601] mpt3sas 0000:65:00.0: irq 109 for MSI/MSI-X
[   21.057733] mpt3sas0-msix0: PCI-MSI-X enabled: IRQ 98
[   21.057734] mpt3sas0-msix1: PCI-MSI-X enabled: IRQ 99
[   21.057734] mpt3sas0-msix2: PCI-MSI-X enabled: IRQ 100
[   21.057734] mpt3sas0-msix3: PCI-MSI-X enabled: IRQ 101
[   21.057735] mpt3sas0-msix4: PCI-MSI-X enabled: IRQ 102
[   21.057735] mpt3sas0-msix5: PCI-MSI-X enabled: IRQ 103
[   21.057735] mpt3sas0-msix6: PCI-MSI-X enabled: IRQ 104
[   21.057736] mpt3sas0-msix7: PCI-MSI-X enabled: IRQ 105
[   21.057736] mpt3sas0-msix8: PCI-MSI-X enabled: IRQ 106
[   21.057736] mpt3sas0-msix9: PCI-MSI-X enabled: IRQ 107
[   21.057737] mpt3sas0-msix10: PCI-MSI-X enabled: IRQ 108
[   21.057737] mpt3sas0-msix11: PCI-MSI-X enabled: IRQ 109
[   21.057738] mpt3sas_cm0: iomem(0x00000000c6100000), mapped(0xffffc223f24c0000), size(65536)
[   21.057739] mpt3sas_cm0: ioport(0x0000000000008000), size(256)
[   21.139655] ata12: SATA link down (SStatus 0 SControl 300)
[   21.141100] ata7: SATA link down (SStatus 0 SControl 300)
[   21.141131] ata10: SATA link down (SStatus 0 SControl 300)
[   21.141606] ata11: SATA link down (SStatus 0 SControl 300)
[   21.142083] ata14: SATA link down (SStatus 0 SControl 300)
[   21.142114] ata9: SATA link down (SStatus 0 SControl 300)
[   21.142567] ata13: SATA link down (SStatus 0 SControl 300)
[   21.147495] ata8: SATA link down (SStatus 0 SControl 300)
[   21.173707] mpt3sas_cm0: Allocated physical memory: size(17684 kB)
[   21.173707] mpt3sas_cm0: Current Controller Queue Depth(9460),Max Controller Queue Depth(9584)
[   21.173708] mpt3sas_cm0: Scatter Gather Elements per IO(128)
[   21.199119] Console: switching to colour frame buffer device 128x48
[   21.219655] mpt3sas_cm0: LSISAS3008: FWVersion(16.00.04.00), ChipRevision(0x02), BiosVersion(18.00.00.00)
[   21.219655] mpt3sas_cm0: Protocol=(
[   21.219656] Initiator
[   21.219656] ,Target
[   21.219656] ), 
[   21.219656] Capabilities=(
[   21.219657] TLR
[   21.219657] ,EEDP
[   21.219657] ,Snapshot Buffer
[   21.219657] ,Diag Trace Buffer
[   21.219658] ,Task Set Full
[   21.219658] ,NCQ
[   21.219658] )
[   21.219815] scsi host14: Fusion MPT SAS Host
[   21.228238] mpt3sas_cm0: sending port enable !!
[   21.229115] mgag200 0000:03:00.0: fb0: mgadrmfb frame buffer device
[   21.580469] [drm] Initialized mgag200 1.0.0 20110418 for 0000:03:00.0 on minor 0
[   21.659287] cryptd: max_cpu_qlen set to 1000
[   21.693072] AVX2 version of gcm_enc/dec engaged.
[   21.697999] AES CTR mode by8 optimization enabled
[   21.768824] alg: No test for __gcm-aes-aesni (__driver-gcm-aes-aesni)
[   21.776248] alg: No test for __generic-gcm-aes-aesni (__driver-generic-gcm-aes-aesni)
[   22.084968] intel_rapl: Found RAPL domain package
[   22.089986] intel_rapl: Found RAPL domain dram
[   22.094889] intel_rapl: DRAM domain energy unit 15300pj
[   22.117274] EDAC MC0: Giving out device to 'skx_edac.c' 'Skylake Socket#0 IMC#0': DEV 0000:64:0a.0
[   22.126671] EDAC MC1: Giving out device to 'skx_edac.c' 'Skylake Socket#0 IMC#1': DEV 0000:64:0c.0
[   22.906075] dcdbas dcdbas: Dell Systems Management Base Driver (version 5.6.0-3.3)
[   22.986264] iTCO_vendor_support: vendor-support=0
[   22.986296] mpt3sas_cm0: host_add: handle(0x0001), sas_addr(0x5d09466034bbc000), phys(8)
[   22.993614] scsi 14:0:0:0: Direct-Access     NETAPP   INF-01-00        0851 PQ: 0 ANSI: 5
[   22.993618] scsi 14:0:0:0: SSP: handle(0x0009), sas_addr(0x500a098d662f7020), phy(0), device_name(0x500a098d662f7020)
[   22.993619] scsi 14:0:0:0: enclosure logical id (0x5d09466034bbc000), slot(0) 
[   22.993620] scsi 14:0:0:0: enclosure level(0x0000), connector name( 0   )
[   22.993622] scsi 14:0:0:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1)
[   23.060815] iTCO_wdt: Intel TCO WatchDog Timer Driver v1.11
[   23.060904] iTCO_wdt: Found a Intel PCH TCO device (Version=4, TCOBASE=0x0400)
[   23.061113] iTCO_wdt: initialized. heartbeat=30 sec (nowayout=0)
[   23.241072] scsi 14:0:1:0: Direct-Access     NETAPP   INF-01-00        0851 PQ: 0 ANSI: 5
[   23.250941] scsi 14:0:1:0: SSP: handle(0x000a), sas_addr(0x500a098d66207020), phy(4), device_name(0x500a098d66207020)
[   23.262934] scsi 14:0:1:0: enclosure logical id (0x5d09466034bbc000), slot(4) 
[   23.270722] scsi 14:0:1:0: enclosure level(0x0000), connector name( 1   )
[   23.277976] scsi 14:0:1:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1)
[   28.349622] mpt3sas_cm0: port enable: SUCCESS
[   28.355653] scsi 14:0:0:0: alua: supports implicit TPGS
[   28.362262] scsi 14:0:0:0: alua: device naa.600a098000d66207000000c65ac4f2c9 port group 1 rel port 8006
[   28.372235] scsi 14:0:0:0: alua: Attached
[   28.376832] scsi 14:0:1:0: alua: supports implicit TPGS
[   28.382509] scsi 14:0:0:0: alua: transition timeout set to 60 seconds
[   28.389398] scsi 14:0:0:0: alua: port group 01 state N non-preferred supports TolUsNA
[   28.389421] scsi 14:0:1:0: alua: device naa.600a098000d66207000000c65ac4f2c9 port group 0 rel port 6
[   28.389426] scsi 14:0:1:0: alua: Attached
[   28.393807] scsi 14:0:1:0: alua: transition timeout set to 60 seconds
[   28.393813] scsi 14:0:1:0: alua: port group 00 state A non-preferred supports TolUsNA
[   28.465156] scsi 14:0:0:0: Attached scsi generic sg0 type 0
[   28.471555] scsi 14:0:1:0: Attached scsi generic sg1 type 0
[   28.492117] sd 14:0:0:0: [sda] 12884901888 512-byte logical blocks: (6.59 TB/6.00 TiB)
[   28.500655] sd 14:0:1:0: [sdb] 12884901888 512-byte logical blocks: (6.59 TB/6.00 TiB)
[   28.500658] sd 14:0:1:0: [sdb] 4096-byte physical blocks
[   28.515025] sd 14:0:0:0: [sda] 4096-byte physical blocks
[   28.520869] sd 14:0:1:0: [sdb] Write Protect is off
[   28.523358] sd 14:0:0:0: [sda] Write Protect is off
[   28.523359] sd 14:0:0:0: [sda] Mode Sense: 83 00 10 08
[   28.523414] sd 14:0:0:0: [sda] Write cache: enabled, read cache: enabled, supports DPO and FUA
[   28.529817] sd 14:0:0:0: [sda] Attached SCSI disk
[   28.548114] sd 14:0:1:0: [sdb] Mode Sense: 83 00 10 08
[   28.548192] sd 14:0:1:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA
[   28.563932] sd 14:0:1:0: [sdb] Attached SCSI disk
[   28.684554] device-mapper: uevent: version 1.0.3
[   28.689718] device-mapper: ioctl: 4.37.1-ioctl (2018-04-03) initialised: dm-devel@redhat.com
[   28.730622] spl: loading out-of-tree module taints kernel.
[   28.730632] spl: loading out-of-tree module taints kernel.
[   28.736703] spl: module verification failed: signature and/or required key missing - tainting kernel
[   28.759693] icp: module license 'CDDL' taints kernel.
[   28.759733] icp: module license 'CDDL' taints kernel.
[   28.759734] Disabling lock debugging due to kernel taint
[   31.815843] ZFS: Loaded module v0.8.1-1, ZFS pool version 5000, ZFS filesystem version 5
[   31.996237] mlx5_ib: Mellanox Connect-IB Infiniband driver v5.0-0
[   32.062180] Loading iSCSI transport class v2.0-870.
[   32.755090] iscsi: registered transport (iser)
[   32.879373] mlx5_core 0000:b3:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(1) RxCqeCmprss(0)
[   32.888903] mlx5_core 0000:b3:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(1) RxCqeCmprss(0)
[   33.142812] RPC: Registered rdma transport module.
[   33.148044] RPC: Registered rdma backchannel transport module.
[   33.202163] ib_srp: srp_sg_tablesize is deprecated, please use cmd_sg_entries
[   33.273529] Rounding down aligned max_sectors from 4294967295 to 4294967288
[   36.760312] systemd-readahead[10700]: Failed to open pack file: Read-only file system
[  147.020883] Process accounting resumed
[  147.041034] nf_conntrack version 0.5.0 (65536 buckets, 262144 max)
[  147.179309] ip6_tables: (C) 2000-2006 Netfilter Core Team
[  147.216460] IPv6: ADDRCONF(NETDEV_UP): eth1: link is not ready
[  147.321049] IPv6: ADDRCONF(NETDEV_UP): eth1: link is not ready
[  147.323116] IPv6: ADDRCONF(NETDEV_UP): eth2: link is not ready
[  147.428262] IPv6: ADDRCONF(NETDEV_UP): eth2: link is not ready
[  147.436381] IPv6: ADDRCONF(NETDEV_UP): eth3: link is not ready
[  147.546633] IPv6: ADDRCONF(NETDEV_UP): eth3: link is not ready
[  147.551694] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready
[  147.600625] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready
[  147.676778] IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready
[  147.685688] IPv6: ADDRCONF(NETDEV_UP): eth1: link is not ready
[  147.693640] IPv6: ADDRCONF(NETDEV_UP): eth2: link is not ready
[  147.700952] IPv6: ADDRCONF(NETDEV_UP): eth3: link is not ready
[  153.218853] Fusion MPT base driver 3.04.20
[  153.223689] Copyright (c) 1999-2008 LSI Corporation
[  153.231299] Fusion MPT misc device (ioctl) driver 3.04.20
[  153.237078] mptctl: Registered with Fusion MPT base driver
[  153.242893] mptctl: /dev/mptctl @ (major,minor=10,220)
[  153.289236] mpt2sas version 20.103.01.00 loaded
[  191.257214] IPv6: ADDRCONF(NETDEV_UP): eth1: link is not ready
[  191.266216] IPv6: ADDRCONF(NETDEV_UP): eth2: link is not ready
[  191.275167] IPv6: ADDRCONF(NETDEV_UP): eth3: link is not ready
[  191.288431] IPv6: ADDRCONF(NETDEV_UP): eth1: link is not ready
[  191.294821] IPv6: ADDRCONF(NETDEV_UP): eth2: link is not ready
[  191.301182] IPv6: ADDRCONF(NETDEV_UP): eth3: link is not ready
[  191.320912] LNet: HW NUMA nodes: 1, HW CPU cores: 12, npartitions: 6
[  191.328342] alg: No test for adler32 (adler32-zlib)
[  191.408446] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready
[  191.415484] IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready
[  192.122685] LNet: Using FastReg for registration
[  192.211874] LNetError: 101:0:(o2iblnd_cb.c:2480:kiblnd_passive_connect()) Can't accept conn from 10.10.32.229@o2ib2 on NA (ib0:0:10.10.33.9): bad dst nid 10.10.33.9@o2ib2
[  192.351876] LNet: Added LNI 10.10.33.9@o2ib2 [8/2556/0/180]
[  201.630421] device-mapper: multipath round-robin: version 1.2.0 loaded
[  201.666356] sd 14:0:1:0: alua: port group 00 state A non-preferred supports TolUsNA
[  201.674121] sd 14:0:1:0: alua: port group 00 state A non-preferred supports TolUsNA
[  272.938585] ZFS: Unloaded module v0.8.1-1
[  278.098214] ZFS: Loaded module v0.8.1-1, ZFS pool version 5000, ZFS filesystem version 5
[  312.881350] Lustre: Lustre: Build Version: 2.12.2_30_g989217d_dirty
[  313.620704] Lustre: f2-MDT0003: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-900
[  318.648707] Lustre: f2-MDT0003: Will be in recovery for at least 2:30, or until 3930 clients reconnect
[  319.655242] Lustre: f2-MDT0003: Connection restored to 3987fee7-8b0f-75c0-9f95-9f204760f382 (at 3309@gni4)
[  319.664889] Lustre: Skipped 13 previous similar messages
[  320.158857] Lustre: f2-MDT0003: Connection restored to 833f6a1f-ad2b-ead5-52ad-a676e2f5eaa1 (at 1708@gni4)
[  320.168559] Lustre: Skipped 154 previous similar messages
[  321.162946] Lustre: f2-MDT0003: Connection restored to 280e934f-9fe3-50fe-9ff3-53210c1f5a01 (at 1760@gni4)
[  321.172655] Lustre: Skipped 129 previous similar messages
[  323.163057] Lustre: f2-MDT0003: Connection restored to dea2a7de-c8c1-6de5-a406-39aed6502e1f (at 1102@gni5)
[  323.172772] Lustre: Skipped 315 previous similar messages
[  327.172424] Lustre: f2-MDT0003: Connection restored to fde8fd65-28a8-f0aa-5169-faccb1cc13c4 (at 3334@gni4)
[  327.182126] Lustre: Skipped 278 previous similar messages
[  335.155768] Lustre: f2-MDT0003: Connection restored to 1856b32f-8dbd-4b02-9b57-1fe8b4f68566 (at 578@gni4)
[  335.165393] Lustre: Skipped 339 previous similar messages
[  351.122972] Lustre: f2-MDT0003: Connection restored to 478971a5-6a8b-2ca1-5c55-fddf4ff98740 (at 233@gni4)
[  351.132592] Lustre: Skipped 2093 previous similar messages
[  491.224154] Lustre: f2-MDT0000-osp-MDT0003: Connection restored to 10.10.33.6@o2ib2 (at 10.10.33.6@o2ib2)
[  491.233762] Lustre: Skipped 667 previous similar messages
[  493.512701] Lustre: f2-MDT0003: recovery is timed out, evict stale exports
[  493.520072] Lustre: f2-MDT0003: disconnecting 1 stale clients
[  497.320639] Lustre: f2-MDT0003: Recovery over after 2:59, of 3930 clients 3929 recovered and 1 was evicted.
[ 2589.036714] Lustre: f2-MDT0003: haven't heard from client e839b884-2bf6-5a17-1c99-f3871d7ef07d (at 10.10.33.4@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fde8c1ce400, cur 1562674535 expire 1562674385 last 1562674308
[ 2604.146379] Lustre: f2-MDT0003: Connection restored to e839b884-2bf6-5a17-1c99-f3871d7ef07d (at 10.10.33.4@o2ib2)
[ 2604.158508] Lustre: Skipped 3 previous similar messages
[ 3834.141641] INFO: task txg_quiesce:37789 blocked for more than 120 seconds.
[ 3834.150496] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3834.160234] txg_quiesce     D ffff9fe13304e180     0 37789      2 0x00000000
[ 3834.167905] Call Trace:
[ 3834.170928]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3834.176475]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3834.183236]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3834.189662]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3834.195865]  [<ffffffffc12f163b>] txg_quiesce_thread+0x2cb/0x3c0 [zfs]
[ 3834.203026]  [<ffffffffc12f1370>] ? txg_init+0x2b0/0x2b0 [zfs]
[ 3834.209450]  [<ffffffffc09f4b93>] thread_generic_wrapper+0x73/0x80 [spl]
[ 3834.216742]  [<ffffffffc09f4b20>] ? __thread_exit+0x20/0x20 [spl]
[ 3834.223423]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3834.228893]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3834.235559]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3834.242565]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3834.249250] INFO: task mdt01_004:40834 blocked for more than 120 seconds.
[ 3834.256590] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3834.264977] mdt01_004       D ffff9fdf273ca080     0 40834      2 0x00000000
[ 3834.272646] Call Trace:
[ 3834.275681]  [<ffffffffc1acfc59>] ? lod_sub_declare_xattr_set+0xf9/0x300 [lod]
[ 3834.283467]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3834.289000]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3834.295744]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3834.302129]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3834.308270]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3834.314742]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3834.321279]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3834.328470]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 3834.335532]  [<ffffffffc1b27af2>] ? mdd_declare_create+0x482/0xcb0 [mdd]
[ 3834.342748]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 3834.349353]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 3834.355937]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 3834.362359]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 3834.369219]  [<ffffffffc10702b8>] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass]
[ 3834.377181]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3834.383671]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3834.390662]  [<ffffffffc19d8706>] ? mdt_intent_fixup_resent+0x36/0x220 [mdt]
[ 3834.398169]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3834.404819]  [<ffffffffc103b129>] ? lprocfs_counter_add+0xf9/0x160 [obdclass]
[ 3834.412399]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3834.419283]  [<ffffffffc19d88f0>] ? mdt_intent_fixup_resent+0x220/0x220 [mdt]
[ 3834.426890]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3834.434037]  [<ffffffffc0de4033>] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs]
[ 3834.441608]  [<ffffffffc0de77be>] ? cfs_hash_add+0xbe/0x1a0 [libcfs]
[ 3834.448416]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3834.455890]  [<ffffffffc16ce6d0>] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc]
[ 3834.463793]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3834.470311]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3834.477596]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3834.485561]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3834.493021]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3834.501086]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3834.508269]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3834.514934]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3834.521183]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3834.527869]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3834.535657]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3834.540940]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3834.547433]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3834.554273]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3834.560767] INFO: task mdt05_017:40854 blocked for more than 120 seconds.
[ 3834.567957] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3834.576187] mdt05_017       D ffff9fdf33b14100     0 40854      2 0x00000000
[ 3834.583689] Call Trace:
[ 3834.586554]  [<ffffffffc1acfc59>] ? lod_sub_declare_xattr_set+0xf9/0x300 [lod]
[ 3834.594181]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3834.599551]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3834.606131]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3834.612366]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3834.618352]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3834.624686]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3834.631094]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3834.638124]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 3834.645047]  [<ffffffffc1b27af2>] ? mdd_declare_create+0x482/0xcb0 [mdd]
[ 3834.652151]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 3834.658649]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 3834.665142]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 3834.671463]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 3834.678227]  [<ffffffffc10702b8>] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass]
[ 3834.686115]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3834.692531]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3834.699461]  [<ffffffffc19d8706>] ? mdt_intent_fixup_resent+0x36/0x220 [mdt]
[ 3834.706907]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3834.713499]  [<ffffffffc103b129>] ? lprocfs_counter_add+0xf9/0x160 [obdclass]
[ 3834.721034]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3834.727882]  [<ffffffffc19d88f0>] ? mdt_intent_fixup_resent+0x220/0x220 [mdt]
[ 3834.735428]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3834.742538]  [<ffffffffc0de4033>] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs]
[ 3834.750073]  [<ffffffffc0de77be>] ? cfs_hash_add+0xbe/0x1a0 [libcfs]
[ 3834.756837]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3834.764304]  [<ffffffffc16ce6d0>] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc]
[ 3834.772207]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3834.778725]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3834.786014]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3834.793983]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3834.801450]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3834.809527]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3834.816722]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3834.823395]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3834.829643]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3834.836330]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3834.844126]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3834.849408]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3834.855901]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3834.862738]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3834.869236] INFO: task mdt01_020:40890 blocked for more than 120 seconds.
[ 3834.876424] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3834.884655] mdt01_020       D ffff9fdf5e911040     0 40890      2 0x00000000
[ 3834.892156] Call Trace:
[ 3834.895039]  [<ffffffffc1acfc59>] ? lod_sub_declare_xattr_set+0xf9/0x300 [lod]
[ 3834.902664]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3834.908048]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3834.914634]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3834.920873]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3834.926862]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3834.933202]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3834.939610]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3834.946642]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 3834.953578]  [<ffffffffc1b27af2>] ? mdd_declare_create+0x482/0xcb0 [mdd]
[ 3834.960682]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 3834.967184]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 3834.973675]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 3834.980002]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 3834.986766]  [<ffffffffc10702b8>] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass]
[ 3834.994655]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3835.001076]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3835.008009]  [<ffffffffc19d8706>] ? mdt_intent_fixup_resent+0x36/0x220 [mdt]
[ 3835.015460]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3835.022055]  [<ffffffffc103b129>] ? lprocfs_counter_add+0xf9/0x160 [obdclass]
[ 3835.029597]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3835.036443]  [<ffffffffc19d88f0>] ? mdt_intent_fixup_resent+0x220/0x220 [mdt]
[ 3835.043995]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3835.051102]  [<ffffffffc0de4033>] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs]
[ 3835.058646]  [<ffffffffc0de77be>] ? cfs_hash_add+0xbe/0x1a0 [libcfs]
[ 3835.065418]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3835.072881]  [<ffffffffc16ce6d0>] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc]
[ 3835.080792]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3835.087305]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3835.094601]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3835.102574]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3835.110040]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3835.118115]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3835.125312]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3835.131985]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3835.138233]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3835.144917]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3835.152719]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3835.158008]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3835.164503]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3835.171337]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3835.177829] INFO: task mdt02_010:40897 blocked for more than 120 seconds.
[ 3835.185013] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3835.193245] mdt02_010       D ffff9fdf334f8000     0 40897      2 0x00000000
[ 3835.200748] Call Trace:
[ 3835.203618]  [<ffffffffc1acfc59>] ? lod_sub_declare_xattr_set+0xf9/0x300 [lod]
[ 3835.211248]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3835.216628]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3835.223215]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3835.229447]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3835.235436]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3835.241769]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3835.248175]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3835.255205]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 3835.262130]  [<ffffffffc1b27af2>] ? mdd_declare_create+0x482/0xcb0 [mdd]
[ 3835.269226]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 3835.275720]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 3835.282216]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 3835.288540]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 3835.295299]  [<ffffffffc10702b8>] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass]
[ 3835.303192]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3835.309608]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3835.316540]  [<ffffffffc19d8706>] ? mdt_intent_fixup_resent+0x36/0x220 [mdt]
[ 3835.323989]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3835.330583]  [<ffffffffc103b129>] ? lprocfs_counter_add+0xf9/0x160 [obdclass]
[ 3835.338118]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3835.344966]  [<ffffffffc19d88f0>] ? mdt_intent_fixup_resent+0x220/0x220 [mdt]
[ 3835.352512]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3835.359617]  [<ffffffffc0de4033>] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs]
[ 3835.367159]  [<ffffffffc0de77be>] ? cfs_hash_add+0xbe/0x1a0 [libcfs]
[ 3835.373928]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3835.381395]  [<ffffffffc16ce6d0>] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc]
[ 3835.389299]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3835.395817]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3835.403113]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3835.411084]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3835.418553]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3835.426629]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3835.433824]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3835.440493]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3835.446743]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3835.453428]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3835.461217]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3835.466502]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3835.472996]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3835.479832]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3835.486333] INFO: task mdt03_029:46557 blocked for more than 120 seconds.
[ 3835.493516] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3835.501748] mdt03_029       D ffff9fde28b55140     0 46557      2 0x00000000
[ 3835.509252] Call Trace:
[ 3835.512117]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3835.517486]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3835.524067]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3835.530306]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3835.536284]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3835.542614]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3835.549017]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3835.556032]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3835.562788]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 3835.569201]  [<ffffffffc19caba2>] ? mdt_pack_acl2body+0x1c2/0x9f0 [mdt]
[ 3835.576219]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 3835.582885]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 3835.589982]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 3835.596653]  [<ffffffffc10702b8>] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass]
[ 3835.604538]  [<ffffffffc107528e>] ? lu_ucred+0x1e/0x30 [obdclass]
[ 3835.611044]  [<ffffffffc19dbf75>] ? mdt_ucred+0x15/0x20 [mdt]
[ 3835.617201]  [<ffffffffc19dc811>] ? mdt_root_squash+0x21/0x430 [mdt]
[ 3835.623965]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3835.630376]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3835.637307]  [<ffffffffc19d8706>] ? mdt_intent_fixup_resent+0x36/0x220 [mdt]
[ 3835.644757]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3835.651349]  [<ffffffffc103b129>] ? lprocfs_counter_add+0xf9/0x160 [obdclass]
[ 3835.658883]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3835.665730]  [<ffffffffc19d88f0>] ? mdt_intent_fixup_resent+0x220/0x220 [mdt]
[ 3835.673277]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3835.680379]  [<ffffffffc0de4033>] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs]
[ 3835.687914]  [<ffffffffc0de77be>] ? cfs_hash_add+0xbe/0x1a0 [libcfs]
[ 3835.694684]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3835.702150]  [<ffffffffc16ce6d0>] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc]
[ 3835.710044]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3835.716555]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3835.723849]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3835.731811]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3835.739271]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3835.747337]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3835.754521]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3835.761186]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3835.767429]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3835.774111]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3835.781903]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3835.787186]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3835.793679]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3835.800520]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3835.807010] INFO: task mdt05_033:46564 blocked for more than 120 seconds.
[ 3835.814194] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3835.822415] mdt05_033       D ffff9fdf2362a080     0 46564      2 0x00000000
[ 3835.829909] Call Trace:
[ 3835.832769]  [<ffffffffc1acfc59>] ? lod_sub_declare_xattr_set+0xf9/0x300 [lod]
[ 3835.840390]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3835.845762]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3835.852347]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3835.858581]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3835.864571]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3835.870907]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3835.877322]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3835.884360]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 3835.891289]  [<ffffffffc1b27af2>] ? mdd_declare_create+0x482/0xcb0 [mdd]
[ 3835.898398]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 3835.904900]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 3835.911391]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 3835.917716]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 3835.924471]  [<ffffffffc10702b8>] ? upcall_cache_get_entry+0x218/0x8b0 [obdclass]
[ 3835.932352]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3835.938764]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3835.945691]  [<ffffffffc19d8706>] ? mdt_intent_fixup_resent+0x36/0x220 [mdt]
[ 3835.953147]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3835.959737]  [<ffffffffc103b129>] ? lprocfs_counter_add+0xf9/0x160 [obdclass]
[ 3835.967279]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3835.974135]  [<ffffffffc167c384>] ? ldlm_lock_create+0xa4/0xa40 [ptlrpc]
[ 3835.981240]  [<ffffffffc19d88f0>] ? mdt_intent_fixup_resent+0x220/0x220 [mdt]
[ 3835.988789]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3835.995896]  [<ffffffffc0de4033>] ? cfs_hash_bd_add_locked+0x63/0x80 [libcfs]
[ 3836.003438]  [<ffffffffc0de77be>] ? cfs_hash_add+0xbe/0x1a0 [libcfs]
[ 3836.010205]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3836.017669]  [<ffffffffc16ce6d0>] ? lustre_swab_ldlm_lock_desc+0x30/0x30 [ptlrpc]
[ 3836.025568]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3836.032086]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3836.039383]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3836.047353]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3836.054819]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3836.062893]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3836.070079]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3836.076744]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3836.082996]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3836.089677]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3836.097474]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3836.102759]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.109252]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3836.116093]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.122584] INFO: task mdt_rdpg01_004:51712 blocked for more than 120 seconds.
[ 3836.130208] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3836.138443] mdt_rdpg01_004  D ffff9fde581b2080     0 51712      2 0x00000000
[ 3836.145934] Call Trace:
[ 3836.148791]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3836.154160]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3836.160737]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3836.166965]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3836.172952]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3836.179282]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3836.185684]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3836.192699]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3836.199450]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[ 3836.205612]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3836.212908]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3836.220882]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3836.228351]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3836.236433]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3836.243634]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3836.250316]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3836.256577]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3836.263268]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3836.271075]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3836.276374]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.282879]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3836.289733]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.296236] INFO: task mdt_rdpg01_011:51772 blocked for more than 120 seconds.
[ 3836.303864] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3836.312107] mdt_rdpg01_011  D ffff9fde57261040     0 51772      2 0x00000000
[ 3836.319616] Call Trace:
[ 3836.322490]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3836.327877]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3836.334473]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3836.340726]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3836.346728]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3836.353078]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3836.359494]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3836.366519]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3836.373280]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[ 3836.379450]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3836.386746]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3836.394720]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3836.402195]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3836.410266]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3836.417465]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3836.424139]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3836.430397]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3836.437086]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3836.444886]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3836.450168]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.456665]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3836.463500]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.469992] INFO: task mdt_rdpg05_005:51963 blocked for more than 120 seconds.
[ 3836.477609] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3836.485838] mdt_rdpg05_005  D ffff9fe10a699040     0 51963      2 0x00000000
[ 3836.493343] Call Trace:
[ 3836.496207]  [<ffffffff9b767c49>] schedule+0x29/0x70
[ 3836.501596]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3836.508188]  [<ffffffff9b0c2d40>] ? wake_up_atomic_t+0x30/0x30
[ 3836.514454]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3836.520458]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3836.526810]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3836.533233]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3836.540265]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3836.547032]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[ 3836.553204]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3836.560497]  [<ffffffffc170ec11>] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc]
[ 3836.568468]  [<ffffffffc0dd8bde>] ? ktime_get_real_seconds+0xe/0x10 [libcfs]
[ 3836.575939]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3836.584012]  [<ffffffffc16d5695>] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc]
[ 3836.591206]  [<ffffffff9b0d6802>] ? default_wake_function+0x12/0x20
[ 3836.597879]  [<ffffffff9b0cbadb>] ? __wake_up_common+0x5b/0x90
[ 3836.604139]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3836.610831]  [<ffffffffc16db640>] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc]
[ 3836.618628]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3836.623911]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3836.630406]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3836.637244]  [<ffffffff9b0c1ba0>] ? insert_kthread_work+0x40/0x40
[ 3857.793070] LNet: Service thread pid 40811 was inactive for 200.45s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 3857.812751] Pid: 40811, comm: mdt05_006 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 3857.822389] Call Trace:
[ 3857.825288]  [<ffffffffc12937a3>] dnode_hold_impl+0x873/0xcb0 [zfs]
[ 3857.832059]  [<ffffffffc127d256>] dmu_object_alloc_impl+0x216/0x400 [zfs]
[ 3857.839339]  [<ffffffffc127d4f9>] dmu_object_alloc_dnsize+0x39/0x40 [zfs]
[ 3857.846591]  [<ffffffffc18b9662>] __osd_object_create+0x82/0x170 [osd_zfs]
[ 3857.853934]  [<ffffffffc18b99cd>] osd_mkreg+0x7d/0x210 [osd_zfs]
[ 3857.860402]  [<ffffffffc18b5cd6>] osd_create+0x346/0xb20 [osd_zfs]
[ 3857.867040]  [<ffffffffc1acd7e5>] lod_sub_create+0x1f5/0x480 [lod]
[ 3857.873688]  [<ffffffffc1abe049>] lod_create+0x69/0x340 [lod]
[ 3857.879895]  [<ffffffffc1b37ef8>] mdd_create_object_internal+0xb8/0x280 [mdd]
[ 3857.887486]  [<ffffffffc1b217e5>] mdd_create_object+0x75/0x820 [mdd]
[ 3857.894292]  [<ffffffffc1b2bd90>] mdd_create+0xe00/0x14b0 [mdd]
[ 3857.900663]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 3857.907475]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3857.913933]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3857.920902]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3857.927524]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3857.934406]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3857.941582]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3857.949097]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3857.955648]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3857.962972]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3857.971063]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3857.977773]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3857.983083]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3857.989956]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 3857.995368] LustreError: dumping log to /tmp/lustre-log.1562675807.40811
[ 3902.230579] LNet: Service thread pid 203911 was inactive for 200.18s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 3902.250457] Pid: 203911, comm: mdt_rdpg00_024 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 3902.260636] Call Trace:
[ 3902.263553]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3902.270208]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3902.276260]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3902.282695]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3902.289194]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3902.296276]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 3902.303322]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 3902.309883]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 3902.316447]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[ 3902.322913]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[ 3902.329488]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[ 3902.336496]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[ 3902.342696]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3902.350061]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3902.358179]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3902.364901]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3902.370214]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3902.377069]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 3902.382478] LustreError: dumping log to /tmp/lustre-log.1562675851.203911
[ 3902.741356] Pid: 203998, comm: mdt_rdpg03_033 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 3902.752454] Call Trace:
[ 3902.755541]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3902.762372]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3902.768383]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3902.774754]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3902.781202]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3902.788256]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3902.795051]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[ 3902.801243]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3902.808583]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3902.816702]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3902.823427]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3902.828754]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3902.835642]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 3902.841075] LustreError: dumping log to /tmp/lustre-log.1562675852.203998
[ 3902.848518] Pid: 63219, comm: mdt05_041 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 3902.859195] Call Trace:
[ 3902.863118]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3902.869941]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3902.876157]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3902.882557]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3902.889043]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3902.896130]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3902.902940]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 3902.909401]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 3902.916103]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 3902.923224]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 3902.929909]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3902.936332]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3902.943273]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3902.949869]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3902.956727]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3902.963850]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3902.971321]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3902.977848]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3902.985163]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3902.993256]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3902.999965]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3903.005271]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3903.012137]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 3903.252147] LNet: Service thread pid 203889 was inactive for 200.40s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 3903.271942] LNet: Skipped 2 previous similar messages
[ 3903.277596] Pid: 203889, comm: mdt03_076 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 3903.287329] Call Trace:
[ 3903.290233]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 3903.296874]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 3903.302898]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 3903.309280]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 3903.315739]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 3903.322792]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 3903.329596]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 3903.336053]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 3903.342774]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 3903.349921]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 3903.356631]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 3903.363080]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 3903.370050]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 3903.376664]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 3903.383537]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 3903.390679]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 3903.398165]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 3903.404711]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 3903.412024]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 3903.420109]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 3903.426807]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 3903.432108]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 3903.438965]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 3903.444361] LNet: Service thread pid 203923 was inactive for 200.58s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3904.273675] LNet: Service thread pid 51963 was inactive for 200.66s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3904.289454] LNet: Skipped 2 previous similar messages
[ 3904.295561] LustreError: dumping log to /tmp/lustre-log.1562675853.51963
[ 3905.295274] LNet: Service thread pid 63819 was inactive for 200.71s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3905.311252] LNet: Skipped 11 previous similar messages
[ 3905.317069] LustreError: dumping log to /tmp/lustre-log.1562675854.63819
[ 3906.316773] LustreError: dumping log to /tmp/lustre-log.1562675855.63780
[ 3907.849120] LNet: Service thread pid 204167 was inactive for 200.45s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3907.865343] LNet: Skipped 5 previous similar messages
[ 3907.871103] LustreError: dumping log to /tmp/lustre-log.1562675857.204167
[ 3910.402968] LustreError: dumping log to /tmp/lustre-log.1562675859.46564
[ 3911.935289] LNet: Service thread pid 63791 was inactive for 200.32s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3911.951554] LNet: Skipped 3 previous similar messages
[ 3911.957334] LustreError: dumping log to /tmp/lustre-log.1562675861.63791
[ 3912.956893] LustreError: dumping log to /tmp/lustre-log.1562675862.40897
[ 3914.489160] LustreError: dumping log to /tmp/lustre-log.1562675863.63782
[ 3915.510783] LustreError: dumping log to /tmp/lustre-log.1562675864.46558
[ 3916.021505] LustreError: dumping log to /tmp/lustre-log.1562675865.63853
[ 3917.043029] LustreError: dumping log to /tmp/lustre-log.1562675866.40835
[ 3918.575350] LustreError: dumping log to /tmp/lustre-log.1562675867.40892
[ 3919.086122] LustreError: dumping log to /tmp/lustre-log.1562675868.63215
[ 3925.215436] LNet: Service thread pid 63261 was inactive for 200.36s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3925.231411] LNet: Skipped 15 previous similar messages
[ 3925.237367] LustreError: dumping log to /tmp/lustre-log.1562675874.63261
[ 3927.258499] LustreError: dumping log to /tmp/lustre-log.1562675876.203901
[ 3927.769284] LustreError: dumping log to /tmp/lustre-log.1562675877.203794
[ 3928.790847] LustreError: dumping log to /tmp/lustre-log.1562675878.203857
[ 3930.323131] LustreError: dumping log to /tmp/lustre-log.1562675879.40851
[ 3940.538647] LustreError: dumping log to /tmp/lustre-log.1562675889.204108
[ 3941.049394] LustreError: dumping log to /tmp/lustre-log.1562675890.46549
[ 3941.560172] LNet: Service thread pid 46525 was inactive for 200.36s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 3941.576184] LNet: Skipped 14 previous similar messages
[ 3942.070913] LustreError: dumping log to /tmp/lustre-log.1562675891.203891
[ 3957.394071] LustreError: dumping log to /tmp/lustre-log.1562675906.63834
[ 4080.999872] LNet: Service thread pid 211611 was inactive for 260.61s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 4081.016087] LNet: Skipped 2 previous similar messages
[ 4081.021689] LustreError: dumping log to /tmp/lustre-log.1562676030.211611
[ 4182.936797] Lustre: 38938:0:(client.c:2134:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1562675531/real 1562675531]  req@ffff9fdebe526300 x1638580646191984/t0(0) o6->f2-OST0011-osc-MDT0003@10.10.33.50@o2ib2:28/4 lens 544/432 e 24 to 1 dl 1562676132 ref 1 fl Rpc:X/0/ffffffff rc 0/-1
[ 4182.970060] Lustre: f2-OST0011-osc-MDT0003: Connection to f2-OST0011 (at 10.10.33.50@o2ib2) was lost; in progress operations using this service will wait for recovery to complete
[ 4183.008589] Lustre: f2-OST0011-osc-MDT0003: Connection restored to 10.10.33.50@o2ib2 (at 10.10.33.50@o2ib2)
[ 4215.841779] LNet: Service thread pid 63242 was inactive for 360.89s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 4215.862224] Pid: 63242, comm: mdt_rdpg02_007 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4215.872399] Call Trace:
[ 4215.875398]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4215.882142]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4215.888280]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4215.894783]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4215.901355]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4215.908521]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 4215.915637]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 4215.922267]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 4215.928890]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[ 4215.935409]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[ 4215.942022]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[ 4215.949067]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[ 4215.955333]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4215.962764]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4215.970967]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4215.977791]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4215.983193]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4215.990169]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4215.995698] LustreError: dumping log to /tmp/lustre-log.1562676165.63242
[ 4251.021226] Lustre: 63771:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fded4442d00 x1637715407501920/t0(0) o101->a232af5c-b333-e608-fc4a-ab5dac72f6bf@876@gni4:611/0 lens 920/3288 e 22 to 0 dl 1562676206 ref 2 fl Interpret:/0/0 rc 0/0
[ 4296.032839] Lustre: 40898:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd687bec00 x1637729395704688/t0(0) o101->e2ff9dc6-f16f-0030-7994-98c1eb446087@10.10.32.103@o2ib2:656/0 lens 880/3288 e 11 to 0 dl 1562676251 ref 2 fl Interpret:/0/0 rc 0/0
[ 4296.376508] Lustre: f2-MDT0003: Client a232af5c-b333-e608-fc4a-ab5dac72f6bf (at 876@gni4) reconnecting
[ 4296.387908] Lustre: f2-MDT0003: Connection restored to a232af5c-b333-e608-fc4a-ab5dac72f6bf (at 876@gni4)
[ 4297.223948] Lustre: 109299:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdf4f973050 x1637715309358016/t0(0) o35->81f8810d-9a89-2bbf-3622-4d215e58e6dd@1069@gni4:657/0 lens 512/456 e 22 to 0 dl 1562676252 ref 2 fl Interpret:/0/0 rc 0/0
[ 4297.260374] Lustre: 109299:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 4 previous similar messages
[ 4299.991269] Lustre: 63869:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd6567c380 x1637722218128880/t0(0) o101->e7566399-e686-8c37-be89-39e485b4b52e@10.10.32.233@o2ib2:660/0 lens 880/3288 e 22 to 0 dl 1562676255 ref 2 fl Interpret:/0/0 rc 0/0
[ 4300.028580] Lustre: 63869:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 15 previous similar messages
[ 4304.029552] Lustre: 63221:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd64c5b180 x1637715051206064/t0(0) o101->b24ff31f-e517-04aa-098d-ff066a7ea5f8@1028@gni4:664/0 lens 912/3288 e 8 to 0 dl 1562676259 ref 2 fl Interpret:/0/0 rc 0/0
[ 4304.066270] Lustre: 63221:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 6 previous similar messages
[ 4312.030228] Lustre: 40808:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcbb26e780 x1637715056104720/t0(0) o101->a12ee8c9-5e94-2cb2-7bca-15c44849cd11@1255@gni4:672/0 lens 912/3288 e 8 to 0 dl 1562676267 ref 2 fl Interpret:/0/0 rc 0/0
[ 4312.066571] Lustre: 40808:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 12 previous similar messages
[ 4333.977258] Lustre: 203846:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcb7694800 x1637715050705712/t0(0) o101->e47ef03d-c378-b5d3-0bb5-4b46733e4a9b@1017@gni4:694/0 lens 912/3288 e 11 to 0 dl 1562676289 ref 2 fl Interpret:/0/0 rc 0/0
[ 4334.014018] Lustre: 203846:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 16 previous similar messages
[ 4340.993452] Lustre: f2-MDT0003: Client c957846e-4015-074d-1136-8fb2895b3aef (at 10.10.32.102@o2ib2) reconnecting
[ 4341.006092] Lustre: f2-MDT0003: Connection restored to c957846e-4015-074d-1136-8fb2895b3aef (at 10.10.32.102@o2ib2)
[ 4342.304608] Lustre: f2-MDT0003: Client c6201616-e876-76a2-09db-ad4a737d4705 (at 10.10.32.104@o2ib2) reconnecting
[ 4342.317267] Lustre: Skipped 3 previous similar messages
[ 4344.611585] Lustre: f2-MDT0003: Client 89d9e1ea-4551-cac3-98c4-d970cba6d6f6 (at 1084@gni4) reconnecting
[ 4344.623530] Lustre: Skipped 4 previous similar messages
[ 4345.148686] Lustre: f2-MDT0003: Connection restored to 3901680b-9599-23ad-8917-dab4a1d50a4a (at 1087@gni4)
[ 4345.160912] Lustre: Skipped 10 previous similar messages
[ 4348.671473] Lustre: f2-MDT0003: Client 783f92d1-f7cc-a708-cbf5-780d2dd3bb19 (at 1074@gni4) reconnecting
[ 4348.683470] Lustre: Skipped 4 previous similar messages
[ 4354.482385] Lustre: f2-MDT0003: Connection restored to 17cd5f45-5d5c-6a29-d5c0-990591f1c666 (at 1263@gni4)
[ 4354.494695] Lustre: Skipped 12 previous similar messages
[ 4357.255194] Lustre: f2-MDT0003: Client f9ff9eb2-208e-4bf5-c124-e9d86005b7a7 (at 1244@gni4) reconnecting
[ 4357.267246] Lustre: Skipped 13 previous similar messages
[ 4379.303592] Lustre: f2-MDT0003: Client e47ef03d-c378-b5d3-0bb5-4b46733e4a9b (at 1017@gni4) reconnecting
[ 4379.315608] Lustre: Skipped 16 previous similar messages
[ 4379.322355] Lustre: f2-MDT0003: Connection restored to e47ef03d-c378-b5d3-0bb5-4b46733e4a9b (at 1017@gni4)
[ 4379.332948] Lustre: Skipped 20 previous similar messages
[ 4458.966787] LNet: Service thread pid 39031 was inactive for 200.73s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 4458.988822] Pid: 39031, comm: mdt04_001 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4458.998822] Call Trace:
[ 4459.002068]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4459.009051]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4459.015411]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4459.022140]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4459.028935]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4459.036306]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 4459.043420]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 4459.050163]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 4459.057149]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 4459.064547]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 4459.071507]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 4459.078210]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 4459.085414]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 4459.092268]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 4459.099355]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 4459.106742]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 4459.114468]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 4459.121233]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4459.128761]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4459.137052]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4459.143942]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4459.149407]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4459.156428]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4459.161982] LustreError: dumping log to /tmp/lustre-log.1562676409.39031
[ 4653.565501] Lustre: 64053:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd1015de80 x1637730089293200/t0(0) o35->bb1ccf8a-a045-3083-e00c-e07924b1f61e@10.10.32.229@o2ib2:259/0 lens 512/456 e 2 to 0 dl 1562676609 ref 2 fl Interpret:/0/0 rc 0/0
[ 4653.601412] Lustre: 64053:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 4 previous similar messages
[ 4698.136503] Lustre: f2-MDT0003: Client bb1ccf8a-a045-3083-e00c-e07924b1f61e (at 10.10.32.229@o2ib2) reconnecting
[ 4698.148517] Lustre: Skipped 3 previous similar messages
[ 4698.155115] Lustre: f2-MDT0003: Connection restored to bb1ccf8a-a045-3083-e00c-e07924b1f61e (at 10.10.32.229@o2ib2)
[ 4698.166095] Lustre: Skipped 3 previous similar messages
[ 4738.861442] Lustre: 109299:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (4/4), not sending early reply
  req@ffff9fdcb5b56780 x1637732362637312/t0(0) o35->d4f00b7a-48c8-b061-7b81-3c377b9b6832@10.10.32.101@o2ib2:344/0 lens 512/456 e 2 to 0 dl 1562676694 ref 2 fl Interpret:/0/0 rc 0/0
[ 4738.897774] Lustre: 109299:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message
[ 4776.657080] LNet: Service thread pid 63854 was inactive for 362.42s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 4776.677869] Pid: 63854, comm: mdt04_052 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4776.687668] Call Trace:
[ 4776.690717]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4776.697508]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4776.703686]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4776.710222]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4776.716829]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4776.724011]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 4776.731153]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 4776.737820]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 4776.744485]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 4776.750971]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 4776.757896]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 4776.764466]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 4776.771558]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 4776.778298]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 4776.785296]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 4776.792591]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 4776.800225]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 4776.806909]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4776.814361]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4776.822592]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4776.829450]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4776.834902]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4776.841905]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4776.847458] LustreError: dumping log to /tmp/lustre-log.1562676728.63854
[ 4776.855229] Pid: 63741, comm: mdt04_037 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4776.864992] Call Trace:
[ 4776.868008]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4776.874770]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4776.880919]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4776.887442]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4776.894032]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4776.901197]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 4776.908278]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 4776.914902]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 4776.921506]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 4776.927930]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 4776.934780]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 4776.941271]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 4776.948276]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 4776.954924]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 4776.961812]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 4776.968963]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 4776.976450]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 4776.982978]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4776.990279]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4776.998351]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4777.005026]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4777.010290]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4777.017112]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4777.022476] Pid: 52186, comm: mdt04_022 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4777.032054] Call Trace:
[ 4777.034884]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4777.041469]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4777.047439]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4777.053780]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4777.060206]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4777.067232]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 4777.074184]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 4777.080684]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 4777.087186]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 4777.093505]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 4777.100267]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 4777.106684]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 4777.113617]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 4777.120196]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 4777.127033]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 4777.134142]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 4777.141597]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 4777.148114]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4777.155401]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4777.163468]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4777.170142]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4777.175408]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4777.182230]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4777.187581] Pid: 63201, comm: mdt04_023 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4777.197155] Call Trace:
[ 4777.199982]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4777.206575]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4777.212548]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4777.218877]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4777.225302]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4777.232313]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 4777.239264]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 4777.245758]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 4777.252253]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 4777.258570]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 4777.265332]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 4777.271737]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 4777.278664]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 4777.285243]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 4777.292081]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 4777.299187]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 4777.306642]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 4777.313145]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4777.320438]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4777.328506]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4777.335182]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4777.340445]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4777.347267]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4783.103998] Lustre: f2-MDT0003: Client d4f00b7a-48c8-b061-7b81-3c377b9b6832 (at 10.10.32.101@o2ib2) reconnecting
[ 4783.115430] Lustre: Skipped 1 previous similar message
[ 4783.121747] Lustre: f2-MDT0003: Connection restored to d4f00b7a-48c8-b061-7b81-3c377b9b6832 (at 10.10.32.101@o2ib2)
[ 4783.132661] Lustre: Skipped 1 previous similar message
[ 4819.560300] LNet: Service thread pid 46521 was inactive for 764.00s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 4819.579981] LNet: Skipped 3 previous similar messages
[ 4819.585628] Pid: 46521, comm: mdt03_019 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 4819.595276] Call Trace:
[ 4819.598179]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 4819.604826]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 4819.610871]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 4819.617294]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 4819.623806]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 4819.630903]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 4819.637743]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 4819.644236]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 4819.650979]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 4819.658144]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 4819.664880]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 4819.671347]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 4819.678333]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 4819.684972]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 4819.691864]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 4819.699048]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 4819.706573]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 4819.713149]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 4819.720493]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 4819.728617]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 4819.735353]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 4819.740687]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 4819.747577]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 4819.753018] LustreError: dumping log to /tmp/lustre-log.1562676771.46521
[ 4821.603247] LNet: Service thread pid 63368 was inactive for 763.76s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 4821.619120] LustreError: dumping log to /tmp/lustre-log.1562676773.63368
[ 4860.420281] LNet: Service thread pid 46511 was inactive for 412.66s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 4860.436362] LustreError: dumping log to /tmp/lustre-log.1562676811.46511
[ 4893.108288] LNet: Service thread pid 109310 was inactive for 814.53s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 4893.108352] Lustre: 39040:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (3/-475), not sending early reply
  req@ffff9fdcb7859f80 x1638052836172944/t0(0) o35->c6201616-e876-76a2-09db-ad4a737d4705@10.10.32.104@o2ib2:497/0 lens 512/456 e 4 to 0 dl 1562676847 ref 2 fl Interpret:/0/0 rc 0/0
[ 4893.108356] Lustre: 39040:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 3 previous similar messages
[ 4893.167188] LNet: Skipped 1 previous similar message
[ 4893.172728] LustreError: dumping log to /tmp/lustre-log.1562676844.109310
[ 4934.952805] Lustre: f2-MDT0003: Client a232af5c-b333-e608-fc4a-ab5dac72f6bf (at 876@gni4) reconnecting
[ 4934.964021] Lustre: Skipped 3 previous similar messages
[ 4934.970546] Lustre: f2-MDT0003: Connection restored to a232af5c-b333-e608-fc4a-ab5dac72f6bf (at 876@gni4)
[ 4934.980902] Lustre: Skipped 3 previous similar messages
[ 5056.547967] LNet: Service thread pid 204155 was inactive for 914.20s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 5056.564656] LustreError: dumping log to /tmp/lustre-log.1562677008.204155
[ 5277.190368] LNet: Service thread pid 128137 was inactive for 1063.23s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 5277.211305] Pid: 128137, comm: mdt_rdpg00_023 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5277.221843] Call Trace:
[ 5277.224907]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5277.231721]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5277.237920]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5277.244498]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5277.251155]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5277.258404]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5277.265615]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5277.272347]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5277.279072]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[ 5277.285707]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[ 5277.292440]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[ 5277.299600]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[ 5277.305975]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5277.313518]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5277.321828]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5277.328738]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5277.334243]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5277.341309]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5277.346918] LustreError: dumping log to /tmp/lustre-log.1562677229.128137
[ 5336.622576] Lustre: f2-MDT0003: Client bb1ccf8a-a045-3083-e00c-e07924b1f61e (at 10.10.32.229@o2ib2) reconnecting
[ 5336.634784] Lustre: Skipped 76 previous similar messages
[ 5336.642132] Lustre: f2-MDT0003: Connection restored to bb1ccf8a-a045-3083-e00c-e07924b1f61e (at 10.10.32.229@o2ib2)
[ 5336.653385] Lustre: Skipped 76 previous similar messages
[ 5416.112997] LNet: Service thread pid 63831 was inactive for 1164.44s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 5416.134066] Pid: 63831, comm: mdt01_078 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5416.143888] Call Trace:
[ 5416.146967]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5416.153786]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5416.159980]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5416.166537]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5416.173159]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5416.180357]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5416.187504]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5416.194152]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5416.200804]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 5416.207266]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 5416.214167]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5416.220703]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5416.227749]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5416.234440]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5416.241379]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5416.248594]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5416.256143]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5416.262723]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5416.270071]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5416.278180]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5416.284900]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5416.290214]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5416.297100]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5416.302537] LustreError: dumping log to /tmp/lustre-log.1562677369.63831
[ 5479.445318] Lustre: 63213:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (4/-300), not sending early reply
  req@ffff9fdcb557b180 x1637729411035568/t0(0) o36->e2ff9dc6-f16f-0030-7994-98c1eb446087@10.10.32.103@o2ib2:331/0 lens 632/2888 e 0 to 0 dl 1562677436 ref 2 fl Interpret:/0/0 rc 0/0
[ 5479.480464] Lustre: 63213:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 31 previous similar messages
[ 5555.035428] LNet: Service thread pid 203846 was inactive for 1201.68s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 5555.055891] Pid: 203846, comm: mdt03_066 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5555.065712] Call Trace:
[ 5555.068702]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5555.075437]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5555.081564]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5555.088068]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5555.094648]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5555.101805]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 5555.108722]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 5555.115276]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 5555.122070]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 5555.129289]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 5555.136068]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5555.142577]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5555.149589]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5555.156248]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5555.163154]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5555.170357]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5555.177906]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5555.184494]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5555.191860]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5555.200002]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5555.206764]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5555.212124]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5555.219042]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5555.224506] LustreError: dumping log to /tmp/lustre-log.1562677508.203846
[ 5612.238695] Pid: 40898, comm: mdt05_028 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5612.249500] Call Trace:
[ 5612.253541]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5612.260430]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5612.266704]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5612.273162]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5612.279700]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5612.286814]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5612.293895]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5612.300490]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5612.307088]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 5612.313503]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 5612.320361]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5612.326860]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5612.333872]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5612.340530]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5612.347429]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5612.354614]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5612.362111]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5612.368656]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5612.375961]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5612.384036]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5612.390717]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5612.395992]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5612.402824]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5612.408200] LustreError: dumping log to /tmp/lustre-log.1562677565.40898
[ 5612.415455] Pid: 46516, comm: mdt02_018 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5612.425044] Call Trace:
[ 5612.427893]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5612.434485]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5612.440468]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5612.446814]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5612.453244]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5612.460273]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5612.467235]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5612.473744]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5612.480246]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 5612.486575]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 5612.493347]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5612.499768]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5612.506702]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5612.513291]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5612.520128]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5612.527234]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5612.534687]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5612.541197]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5612.548485]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5612.556554]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5612.563227]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5612.568493]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5612.575315]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5612.580678] Pid: 203841, comm: mdt02_062 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5612.590343] Call Trace:
[ 5612.593174]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5612.599756]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5612.605731]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5612.612066]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5612.618492]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5612.625511]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5612.632463]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5612.638964]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5612.645467]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 5612.651787]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 5612.658546]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5612.664961]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5612.671897]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5612.678477]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5612.685314]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5612.692420]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5612.699874]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5612.706384]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5612.713672]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5612.721740]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5612.728414]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5612.733677]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5612.740500]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5612.745854] Pid: 203897, comm: mdt03_079 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5612.755510] Call Trace:
[ 5612.758341]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5612.764925]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5612.770898]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5612.777228]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5612.783643]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5612.790663]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5612.797615]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5612.804108]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5612.810600]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 5612.816920]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 5612.823672]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5612.830079]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5612.837003]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5612.843582]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5612.850422]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5612.857530]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5612.864982]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5612.871483]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5612.878763]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5612.886821]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5612.893495]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5612.898762]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5612.905581]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5612.910937] Pid: 63798, comm: mdt01_069 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 5612.920507] Call Trace:
[ 5612.923336]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 5612.929920]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 5612.935886]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 5612.942215]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 5612.948630]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 5612.955642]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 5612.962584]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 5612.969077]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 5612.975570]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 5612.981891]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 5612.988643]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 5612.995049]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 5613.001974]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 5613.008552]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 5613.015392]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 5613.022498]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 5613.029951]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 5613.036454]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 5613.043742]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 5613.051798]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 5613.058472]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 5613.063740]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 5613.070561]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 5613.075915] LNet: Service thread pid 63722 was inactive for 1200.89s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 5649.012227] LustreError: dumping log to /tmp/lustre-log.1562677602.63221
[ 5673.527915] LustreError: dumping log to /tmp/lustre-log.1562677627.63730
[ 5722.559264] LustreError: dumping log to /tmp/lustre-log.1562677676.63231
[ 5779.762487] LNet: Service thread pid 63771 was inactive for 1201.33s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 5779.778097] LNet: Skipped 18 previous similar messages
[ 5779.783819] LustreError: dumping log to /tmp/lustre-log.1562677733.63771
[ 5818.068211] LustreError: dumping log to /tmp/lustre-log.1562677771.63797
[ 5857.395432] LustreError: dumping log to /tmp/lustre-log.1562677811.63822
[ 5862.905796] Lustre: f2-MDT0003: Client fb2a55e1-4841-b544-3e5f-7e60b05d8032 (at 10.10.32.134@o2ib2) reconnecting
[ 5862.917374] Lustre: Skipped 83 previous similar messages
[ 5862.923522] Lustre: f2-MDT0003: Connection restored to fb2a55e1-4841-b544-3e5f-7e60b05d8032 (at 10.10.32.134@o2ib2)
[ 5862.934438] Lustre: Skipped 83 previous similar messages
[ 5917.436636] LustreError: 63823:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562677571, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fdea6291b00/0xd5b51f242090f0b4 lrc: 3/1,0 mode: --/PR res: [0x1400017075:0x817:0x0].0x0 bits 0x13/0x0 rrc: 31 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 63823 timeout: 0 lvb_type: 0
[ 5917.437634] LustreError: dumping log to /tmp/lustre-log.1562677871.63213
[ 5917.489512] LustreError: 63823:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 13 previous similar messages
[ 6029.515952] Lustre: 63814:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcb3f33600 x1637713535188816/t0(0) o101->5ec2f591-5a26-0a09-8f62-a71e142025b9@790@gni5:128/0 lens 888/3288 e 1 to 0 dl 1562677988 ref 2 fl Interpret:/0/0 rc 0/0
[ 6029.550819] Lustre: 63814:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 5 previous similar messages
[ 6073.011547] LustreError: 63779:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562677727, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde4011ba80/0xd5b51f2420914ff1 lrc: 3/0,1 mode: --/CW res: [0x14000184ba:0x578:0x0].0x0 bits 0x2/0x0 rrc: 31 type: IBT flags: 0x40210400000020 nid: local remote: 0x0 expref: -99 pid: 63779 timeout: 0 lvb_type: 0
[ 6073.058400] LustreError: 63779:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 18 previous similar messages
[ 6131.157755] LNet: Service thread pid 63379 was inactive for 1201.24s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 6131.178869] LNet: Skipped 5 previous similar messages
[ 6131.184610] Pid: 63379, comm: mdt03_034 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6131.194451] Call Trace:
[ 6131.197555]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6131.204405]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6131.210655]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6131.217293]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6131.223996]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6131.231283]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 6131.238537]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 6131.245297]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 6131.252058]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 6131.258645]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 6131.265670]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6131.272349]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6131.279517]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6131.286320]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6131.293385]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6131.300744]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6131.308457]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6131.315206]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6131.322711]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6131.330984]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6131.337874]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6131.343368]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6131.350428]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6131.356018] LustreError: dumping log to /tmp/lustre-log.1562678085.63379
[ 6133.200821] Pid: 63374, comm: mdt00_033 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6133.211981] Call Trace:
[ 6133.216338]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6133.224075]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6133.231814]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6133.239183]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6133.246709]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6133.254145]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6133.261229]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6133.268221]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6133.275457]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6133.283032]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6133.289658]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6133.297054]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6133.305217]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6133.311976]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6133.317320]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6133.324212]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6133.329642] LustreError: dumping log to /tmp/lustre-log.1562678087.63374
[ 6133.336968] Pid: 63840, comm: mdt04_049 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6133.347553] Call Trace:
[ 6133.350817]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6133.357824]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6133.363817]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6133.370189]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6133.376638]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6133.383679]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 6133.390654]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 6133.397163]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 6133.403682]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 6133.410020]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 6133.416797]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6133.423231]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6133.430171]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6133.436769]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6133.443635]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6133.450758]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6133.458238]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6133.464775]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6133.472087]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6133.480180]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6133.486890]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6133.492188]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6133.499043]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6133.504443] Pid: 133919, comm: mdt04_062 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6133.514141] Call Trace:
[ 6133.517008]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6133.524337]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6133.531918]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6133.539125]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6133.546500]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6133.553789]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6133.560732]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6133.567590]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6133.574713]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6133.582185]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6133.588720]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6133.596025]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6133.604117]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6133.610816]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6133.616116]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6133.622972]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6133.628363] Pid: 63675, comm: mdt04_035 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6133.637966] Call Trace:
[ 6133.640831]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6133.648149]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6133.655725]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6133.662934]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6133.670309]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6133.677596]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6133.684539]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6133.691395]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6133.698523]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6133.705997]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6133.712525]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6133.719842]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6133.727934]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6133.734643]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6133.739950]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6133.746814]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6133.752224] LNet: Service thread pid 46559 was inactive for 360.88s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 6133.765926] LNet: Skipped 18 previous similar messages
[ 6135.243859] LustreError: dumping log to /tmp/lustre-log.1562678089.46517
[ 6155.674418] LustreError: dumping log to /tmp/lustre-log.1562678110.39027
[ 6372.236848] LustreError: dumping log to /tmp/lustre-log.1562678327.46537
[ 6453.958185] Pid: 63837, comm: mdt05_079 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6453.968867] Call Trace:
[ 6453.972788]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6453.979633]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6453.985858]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6453.992282]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6453.998766]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6454.005831]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 6454.012642]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 6454.019108]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 6454.025826]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 6454.032967]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 6454.039677]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6454.046128]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6454.053096]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6454.059721]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6454.066609]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6454.073795]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6454.081316]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6454.087870]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6454.095202]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6454.103310]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6454.110027]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6454.115345]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6454.122218]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6454.127632] LustreError: dumping log to /tmp/lustre-log.1562678409.63837
[ 6454.135204] Pid: 203882, comm: mdt03_074 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6454.145902] Call Trace:
[ 6454.149190]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6454.156224]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6454.162243]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6454.168628]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6454.175098]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6454.182150]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 6454.188956]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 6454.195407]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 6454.202115]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 6454.209254]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 6454.215956]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6454.222397]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6454.229355]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6454.235973]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6454.242844]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6454.249970]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6454.257432]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6454.263947]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6454.271247]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6454.279332]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6454.286034]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6454.291324]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6454.298169]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6502.990935] Pid: 46509, comm: mdt03_017 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6503.001456] Call Trace:
[ 6503.004968]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6503.011977]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6503.018004]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6503.024374]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6503.030825]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6503.037854]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 6503.044641]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 6503.051072]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 6503.057766]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 6503.064888]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 6503.071580]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6503.078004]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6503.084947]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6503.091546]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6503.098399]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6503.105552]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6503.113038]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6503.119565]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6503.126870]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6503.134950]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6503.141646]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6503.146928]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6503.153759]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6503.159139] LustreError: dumping log to /tmp/lustre-log.1562678458.46509
[ 6532.352336] Lustre: f2-MDT0003: Client 6b380e87-16ee-2837-6f80-d86a273b02ae (at 10.10.32.129@o2ib2) reconnecting
[ 6532.362891] Lustre: Skipped 136 previous similar messages
[ 6532.368702] Lustre: f2-MDT0003: Connection restored to 6b380e87-16ee-2837-6f80-d86a273b02ae (at 10.10.32.129@o2ib2)
[ 6532.379551] Lustre: Skipped 136 previous similar messages
[ 6629.147711] Lustre: 63843:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd4305d100 x1637721496847536/t0(0) o101->30dfd750-cdf7-d562-e28c-e616526d810f@10.10.32.132@o2ib2:729/0 lens 880/3288 e 1 to 0 dl 1562678589 ref 2 fl Interpret:/0/0 rc 0/0
[ 6629.182466] Lustre: 63843:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 50 previous similar messages
[ 6633.744489] Pid: 63375, comm: mdt03_030 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6633.755298] Call Trace:
[ 6633.759358]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6633.766256]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6633.772419]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6633.778878]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6633.785413]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6633.792538]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 6633.799629]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 6633.806225]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 6633.812821]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 6633.819244]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 6633.826109]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6633.832612]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6633.839615]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6633.846273]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6633.853188]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6633.860400]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6633.867928]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6633.874517]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6633.881885]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6633.890027]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6633.896788]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6633.902149]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6633.909076]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6633.914539] LustreError: dumping log to /tmp/lustre-log.1562678589.63375
[ 6662.346994] LNet: Service thread pid 46561 was inactive for 1201.89s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 6662.367226] LNet: Skipped 8 previous similar messages
[ 6662.372979] Pid: 46561, comm: mdt05_032 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6662.382691] Call Trace:
[ 6662.385683]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6662.392414]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6662.398532]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6662.405018]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6662.411588]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6662.418737]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 6662.425843]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 6662.432466]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 6662.439080]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 6662.445511]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 6662.452377]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6662.458880]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6662.465882]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6662.472531]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6662.479422]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6662.486588]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6662.494084]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6662.500619]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6662.507926]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6662.516000]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6662.522693]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6662.527993]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6662.534850]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6662.540255] LustreError: dumping log to /tmp/lustre-log.1562678618.46561
[ 6746.110562] LNet: Service thread pid 63737 was inactive for 763.68s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 6746.126158] LNet: Skipped 10 previous similar messages
[ 6746.132102] LustreError: dumping log to /tmp/lustre-log.1562678702.63737
[ 6817.616104] Pid: 40824, comm: mdt05_012 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6817.626793] Call Trace:
[ 6817.630177]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6817.638070]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6817.645713]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6817.652994]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6817.660444]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6817.667791]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6817.674797]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6817.681705]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6817.688888]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6817.696410]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6817.703000]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6817.710363]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6817.718501]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6817.725243]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6817.730570]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6817.737451]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6817.742875] LustreError: dumping log to /tmp/lustre-log.1562678774.40824
[ 6817.750187] Pid: 63213, comm: mdt05_038 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6817.759811] Call Trace:
[ 6817.762684]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6817.770025]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6817.777619]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6817.784865]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6817.792284]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6817.799614]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6817.806603]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6817.813501]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6817.820668]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6817.828188]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6817.834771]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6817.842135]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6817.850271]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6817.857022]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6817.862376]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6817.869282]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6817.874732] Pid: 203893, comm: mdt03_078 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6817.884473] Call Trace:
[ 6817.887381]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6817.894753]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6817.902379]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6817.909648]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6817.917084]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6817.924423]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6817.931413]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6817.938312]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6817.945468]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6817.952965]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6817.959511]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6817.966835]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6817.974943]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6817.981661]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6817.986978]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6817.993851]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6817.999264] Pid: 46527, comm: mdt03_021 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6818.008895] Call Trace:
[ 6818.011778]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 6818.018413]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 6818.024439]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 6818.030846]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 6818.037330]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 6818.044394]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 6818.051387]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 6818.057931]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 6818.064469]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 6818.070833]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 6818.077622]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 6818.084062]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 6818.091010]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 6818.097607]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6818.104456]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6818.111580]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6818.119051]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6818.125576]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6818.132883]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6818.140968]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6818.147668]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6818.152959]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6818.159815]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6818.165202] Pid: 39029, comm: mdt03_002 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 6818.174808] Call Trace:
[ 6818.177671]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 6818.184989]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 6818.192565]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 6818.199784]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 6818.207157]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 6818.214445]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 6818.221389]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 6818.228235]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 6818.235360]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 6818.242831]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 6818.249351]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 6818.256655]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 6818.264740]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 6818.271442]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 6818.276730]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 6818.283588]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 6895.250598] LustreError: dumping log to /tmp/lustre-log.1562678851.40830
[ 6972.884978] LustreError: dumping log to /tmp/lustre-log.1562678929.63207
[ 7050.519286] LustreError: dumping log to /tmp/lustre-log.1562679007.40826
[ 7107.723434] LustreError: dumping log to /tmp/lustre-log.1562679064.63745
[ 7119.981463] Pid: 46545, comm: mdt00_021 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7119.991991] Call Trace:
[ 7119.995413]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7120.002442]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7120.008433]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7120.014801]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7120.021248]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7120.028271]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 7120.035263]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 7120.041763]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 7120.048274]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 7120.054603]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 7120.061381]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7120.067805]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7120.074741]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7120.081335]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7120.088182]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7120.095324]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7120.102803]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7120.109329]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7120.116627]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7120.124703]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7120.131385]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7120.136667]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7120.143515]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7120.148911] LustreError: dumping log to /tmp/lustre-log.1562679077.46545
[ 7154.602021] Lustre: f2-MDT0003: Client e2ff9dc6-f16f-0030-7994-98c1eb446087 (at 10.10.32.103@o2ib2) reconnecting
[ 7154.612572] Lustre: Skipped 143 previous similar messages
[ 7154.618372] Lustre: f2-MDT0003: Connection restored to e2ff9dc6-f16f-0030-7994-98c1eb446087 (at 10.10.32.103@o2ib2)
[ 7154.629189] Lustre: Skipped 143 previous similar messages
[ 7292.967369] Lustre: 203831:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-73), not sending early reply
  req@ffff9fdd4acdc800 x1637720419064368/t0(0) o101->37723e5a-ef97-8aea-4a01-ee00091bb44c@10.10.32.135@o2ib2:640/0 lens 880/3288 e 0 to 0 dl 1562679255 ref 2 fl Interpret:/0/0 rc 0/0
[ 7293.002247] Lustre: 203831:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 10 previous similar messages
[ 7352.888525] LNet: Service thread pid 63849 was inactive for 1164.20s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 7352.908677] LNet: Skipped 6 previous similar messages
[ 7352.914286] Pid: 63849, comm: mdt02_054 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7352.923990] Call Trace:
[ 7352.926940]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7352.933639]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7352.939724]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7352.946194]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7352.952730]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7352.959862]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 7352.966960]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 7352.973575]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 7352.980180]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 7352.986604]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 7352.993466]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7352.999979]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7353.007001]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7353.013680]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7353.020616]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7353.027844]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7353.035411]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7353.042030]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7353.049425]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7353.057596]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7353.064388]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7353.069783]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7353.076736]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7353.082225] LustreError: dumping log to /tmp/lustre-log.1562679310.63849
[ 7353.090286] Pid: 40879, comm: mdt03_013 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7353.090287] Call Trace:
[ 7353.090309]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7353.090316]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7353.090363]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7353.090408]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7353.090420]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7353.090443]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7353.090465]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7353.090486]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7353.090507]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7353.090527]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7353.090546]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7353.090563]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7353.090580]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7353.090597]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7353.090637]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7353.090684]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7353.090752]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7353.090820]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7353.090877]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7353.090932]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7353.090937]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7353.090942]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7353.090961]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7360.039463] Pid: 63835, comm: mdt05_077 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7360.049912] Call Trace:
[ 7360.053606]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7360.060344]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7360.066410]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7360.072750]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7360.079173]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7360.086192]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7360.092968]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7360.099399]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7360.106091]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7360.113211]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7360.119895]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7360.126317]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7360.133252]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7360.139839]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7360.146687]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7360.153804]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7360.161265]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7360.167781]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7360.175078]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7360.183158]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7360.189845]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7360.195120]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7360.201943]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7360.207303] LustreError: dumping log to /tmp/lustre-log.1562679317.63835
[ 7450.958255] Pid: 63811, comm: mdt00_049 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7450.968748] Call Trace:
[ 7450.972113]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7450.979230]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7450.985210]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7450.991575]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7450.998022]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7451.005055]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 7451.012038]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 7451.018657]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 7451.025181]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 7451.031544]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 7451.038347]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7451.044805]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7451.051766]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7451.058380]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7451.065255]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7451.072409]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7451.079916]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7451.086457]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7451.093760]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7451.101834]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7451.108518]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7451.113791]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7451.120619]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7451.126000] LustreError: dumping log to /tmp/lustre-log.1562679408.63811
[ 7585.803214] Pid: 203888, comm: mdt04_076 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7585.813808] Call Trace:
[ 7585.817375]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7585.824420]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7585.830463]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7585.836827]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7585.843267]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7585.850298]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 7585.857291]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 7585.863806]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 7585.870323]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 7585.876667]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[ 7585.883446]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7585.889870]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7585.896802]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7585.903391]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7585.910230]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7585.917362]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7585.924832]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7585.931350]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7585.938647]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7585.946716]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7585.953390]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7585.958654]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7585.965475]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7585.970845] LustreError: dumping log to /tmp/lustre-log.1562679544.203888
[ 7712.474927] Pid: 40828, comm: mdt04_008 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7712.485418] Call Trace:
[ 7712.488968]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7712.495978]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7712.502021]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7712.508393]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7712.514843]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7712.521873]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7712.528660]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7712.535095]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7712.541792]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7712.548915]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7712.555600]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7712.562025]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7712.568969]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7712.575573]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7712.582418]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7712.589570]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7712.597057]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7712.603585]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7712.610893]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7712.618973]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7712.625658]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7712.630930]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7712.637761]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7712.643141] LustreError: dumping log to /tmp/lustre-log.1562679671.40828
[ 7724.733340] Pid: 63736, comm: mdt03_037 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7724.743848] Call Trace:
[ 7724.746979]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7724.753866]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7724.759859]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7724.766222]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7724.772673]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7724.779708]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7724.786496]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7724.792929]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7724.799619]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7724.806745]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7724.813436]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7724.819867]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7724.826811]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7724.833407]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7724.840265]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7724.847406]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7724.854883]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7724.861411]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7724.868716]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7724.876793]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7724.883483]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7724.888759]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7724.895598]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7724.900976] LustreError: dumping log to /tmp/lustre-log.1562679683.63736
[ 7736.991898] Pid: 46523, comm: mdt04_016 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7737.002423] Call Trace:
[ 7737.006179]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7737.012940]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7737.019104]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7737.025477]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7737.031925]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7737.038963]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7737.045751]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7737.052190]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7737.058882]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7737.066007]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7737.072699]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7737.079135]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7737.086082]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7737.092679]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7737.099534]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7737.106684]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7737.114175]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7737.120708]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7737.128015]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7737.136098]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7737.142783]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7737.148065]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7737.154902]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7737.160289] LustreError: dumping log to /tmp/lustre-log.1562679695.46523
[ 7760.911687] Lustre: f2-MDT0003: Client 30dfd750-cdf7-d562-e28c-e616526d810f (at 10.10.32.132@o2ib2) reconnecting
[ 7760.923168] Lustre: Skipped 144 previous similar messages
[ 7760.929660] Lustre: f2-MDT0003: Connection restored to 30dfd750-cdf7-d562-e28c-e616526d810f (at 10.10.32.132@o2ib2)
[ 7760.940663] Lustre: Skipped 144 previous similar messages
[ 7794.198068] Pid: 46572, comm: mdt04_021 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7794.208715] Call Trace:
[ 7794.212531]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7794.219346]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7794.225373]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7794.231781]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7794.238274]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7794.245343]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7794.252177]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7794.258650]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7794.265378]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7794.272537]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7794.279262]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7794.285721]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7794.292697]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7794.299329]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7794.306220]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7794.313405]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7794.320918]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7794.327464]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7794.334778]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7794.342879]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7794.349586]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7794.354896]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7794.361772]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7794.367201] LustreError: dumping log to /tmp/lustre-log.1562679753.46572
[ 7818.715041] Pid: 63377, comm: mdt03_032 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7818.725676] Call Trace:
[ 7818.729457]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7818.736283]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7818.742321]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7818.748732]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7818.755217]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7818.762293]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7818.769119]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7818.775586]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7818.782313]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7818.789471]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7818.796191]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7818.802649]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7818.809627]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7818.816257]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7818.823138]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7818.830325]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7818.837836]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7818.844373]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7818.851688]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7818.859773]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7818.866462]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7818.871743]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7818.878584]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7818.883971] LustreError: dumping log to /tmp/lustre-log.1562679777.63377
[ 7849.361076] Pid: 63829, comm: mdt05_074 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 7849.371609] Call Trace:
[ 7849.375375]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 7849.382143]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 7849.388135]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 7849.394514]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 7849.400963]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 7849.408001]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 7849.414798]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 7849.421238]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 7849.427929]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 7849.435055]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 7849.441754]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 7849.448179]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 7849.455120]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 7849.461726]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 7849.468584]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 7849.475733]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 7849.483221]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 7849.489757]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 7849.497070]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 7849.505157]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 7849.511859]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 7849.517145]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 7849.523985]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 7849.529364] LustreError: dumping log to /tmp/lustre-log.1562679808.63829
[ 8104.744194] LNet: Service thread pid 46532 was inactive for 1204.53s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 8104.763866] LNet: Skipped 10 previous similar messages
[ 8104.769415] Pid: 46532, comm: mdt04_018 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 8104.779043] Call Trace:
[ 8104.781925]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 8104.788558]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 8104.794573]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 8104.800978]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 8104.807466]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 8104.814527]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 8104.821349]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 8104.827815]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 8104.834543]  [<ffffffffc19fb05d>] mdt_reint_open+0x1acd/0x27d0 [mdt]
[ 8104.841346]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 8104.847806]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 8104.854774]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 8104.861401]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 8104.868287]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 8104.875464]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 8104.882976]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 8104.889531]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 8104.896852]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 8104.904946]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 8104.911647]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 8104.916947]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 8104.923811]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 8104.929225] LustreError: dumping log to /tmp/lustre-log.1562680064.46532
[ 8318.148405] Lustre: 63233:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-351), not sending early reply
  req@ffff9fdd3474ad00 x1637721441740752/t0(0) o101->6b380e87-16ee-2837-6f80-d86a273b02ae@10.10.32.129@o2ib2:158/0 lens 880/3288 e 1 to 0 dl 1562680283 ref 2 fl Interpret:/0/0 rc 0/0
[ 8318.183430] Lustre: 63233:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 4 previous similar messages
[ 8363.827709] Lustre: f2-MDT0003: Client a15eebf4-4358-c5a6-56de-109f68847539 (at 305@gni5) reconnecting
[ 8363.838591] Lustre: Skipped 143 previous similar messages
[ 8363.845266] Lustre: f2-MDT0003: Connection restored to a15eebf4-4358-c5a6-56de-109f68847539 (at 305@gni5)
[ 8363.855351] Lustre: Skipped 143 previous similar messages
[ 8407.120209] Pid: 63235, comm: mdt_rdpg05_009 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 8407.131532] Call Trace:
[ 8407.135053]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 8407.142343]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 8407.148470]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 8407.154967]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 8407.161532]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 8407.168688]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 8407.175796]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 8407.182426]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 8407.189051]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[ 8407.195577]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[ 8407.202187]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[ 8407.209227]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[ 8407.215473]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 8407.222882]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 8407.231060]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 8407.237820]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 8407.243190]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 8407.250114]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 8407.255587] LustreError: dumping log to /tmp/lustre-log.1562680367.63235
[ 8558.310686] Pid: 46514, comm: mdt02_016 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 8558.321455] Call Trace:
[ 8558.324636]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 8558.331571]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 8558.337656]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 8558.344111]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 8558.350650]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 8558.357772]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 8558.364647]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 8558.371165]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 8558.377935]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 8558.385138]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 8558.391899]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 8558.398400]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 8558.405420]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 8558.412096]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 8558.419023]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 8558.426257]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 8558.433811]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 8558.440410]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 8558.447767]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 8558.455886]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 8558.462603]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 8558.467914]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 8558.474784]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 8558.480191] LustreError: dumping log to /tmp/lustre-log.1562680518.46514
[ 8909.721661] LNet: Service thread pid 40853 was inactive for 200.34s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 8909.741244] LNet: Skipped 2 previous similar messages
[ 8909.746855] Pid: 40853, comm: mdt01_009 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 8909.756467] Call Trace:
[ 8909.759345]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 8909.765966]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 8909.771975]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 8909.778374]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 8909.784851]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 8909.791913]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 8909.798726]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 8909.805194]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 8909.811910]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 8909.819062]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 8909.825783]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 8909.832236]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 8909.839204]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 8909.845828]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 8909.852701]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 8909.859887]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 8909.867400]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 8909.873952]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 8909.881269]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 8909.889361]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 8909.896069]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 8909.901367]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 8909.908234]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 8909.913646] LustreError: dumping log to /tmp/lustre-log.1562680871.40853
[ 8962.841661] Pid: 63843, comm: mdt04_050 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 8962.852265] Call Trace:
[ 8962.855464]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 8962.862274]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 8962.868292]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 8962.874690]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 8962.881172]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 8962.888236]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 8962.895059]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 8962.901525]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 8962.908242]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 8962.915392]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 8962.922112]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 8962.928561]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 8962.935530]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 8962.942150]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 8962.949033]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 8962.956210]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 8962.963727]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 8962.970277]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 8962.977588]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 8962.985670]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 8962.992365]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 8962.997649]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 8963.004487]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 8963.009875] LustreError: dumping log to /tmp/lustre-log.1562680924.63843
[ 8963.017305] Pid: 203900, comm: mdt04_078 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 8963.027925] Call Trace:
[ 8963.031366]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 8963.038414]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 8963.044404]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 8963.050759]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 8963.057200]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 8963.064230]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 8963.071006]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 8963.077441]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 8963.084132]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 8963.091255]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 8963.097951]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 8963.104379]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 8963.111323]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 8963.117920]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 8963.124775]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 8963.131903]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 8963.139381]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 8963.145906]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 8963.153212]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 8963.161297]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 8963.167979]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 8963.173251]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 8963.180090]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 8966.408581] Lustre: f2-MDT0003: Client 96cc3be9-ba47-69f3-7ec2-1712acf4c671 (at 587@gni5) reconnecting
[ 8966.419181] Lustre: Skipped 120 previous similar messages
[ 8966.424230] Lustre: f2-MDT0003: Connection restored to 71e4777a-3114-f185-255b-e15cebd80d08 (at 591@gni5)
[ 8966.424233] Lustre: Skipped 120 previous similar messages
[ 9114.028186] Pid: 40840, comm: mdt05_014 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9114.038787] Call Trace:
[ 9114.042444]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9114.049153]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9114.055262]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9114.061669]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9114.068161]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9114.075234]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9114.082056]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9114.088525]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9114.095262]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9114.102426]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9114.109151]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9114.115610]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9114.122588]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9114.129211]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9114.136100]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9114.143286]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9114.150808]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9114.157360]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9114.164683]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9114.172785]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9114.179503]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9114.184812]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9114.191686]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9114.197100] LustreError: dumping log to /tmp/lustre-log.1562681076.40840
[ 9153.867803] Pid: 40856, comm: mdt01_010 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9153.878406] Call Trace:
[ 9153.881502]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9153.888356]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9153.894374]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9153.900779]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9153.907260]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9153.914328]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9153.921141]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9153.927607]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9153.934327]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9153.941476]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9153.948191]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9153.954643]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9153.961612]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9153.968233]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9153.975106]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9153.982283]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9153.989787]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9153.996331]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9154.003646]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9154.011730]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9154.018422]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9154.023707]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9154.030543]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9154.035932] LustreError: dumping log to /tmp/lustre-log.1562681115.40856
[ 9162.040118] LNet: Service thread pid 63814 was inactive for 200.20s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 9162.055606] LNet: Skipped 17 previous similar messages
[ 9162.061543] LustreError: dumping log to /tmp/lustre-log.1562681124.63814
[ 9224.353225] Pid: 40904, comm: mdt01_022 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9224.363839] Call Trace:
[ 9224.366997]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9224.373911]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9224.379938]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9224.386344]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9224.392829]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9224.399900]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[ 9224.406930]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[ 9224.413482]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[ 9224.420027]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[ 9224.426392]  [<ffffffffc19eac04>] mdt_create+0xb54/0x1090 [mdt]
[ 9224.432762]  [<ffffffffc19eb2ab>] mdt_reint_create+0x16b/0x360 [mdt]
[ 9224.439560]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9224.446000]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9224.452959]  [<ffffffffc19d7497>] mdt_reint+0x67/0x140 [mdt]
[ 9224.459054]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9224.466395]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9224.474495]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9224.481189]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9224.486483]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9224.493347]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9224.498758] LustreError: dumping log to /tmp/lustre-log.1562681186.40904
[ 9289.942312] Lustre: 39026:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-334), not sending early reply
  req@ffff9fdd50b45e80 x1637723724972080/t0(0) o101->6d30d5c4-7610-a482-dfbe-2e75a61b7d56@10.10.32.130@o2ib2:377/0 lens 880/3288 e 17 to 0 dl 1562681257 ref 2 fl Interpret:/0/0 rc 0/0
[ 9289.977246] Lustre: 39026:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 5 previous similar messages
[ 9306.075180] Pid: 40855, comm: mdt05_018 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9306.085955] Call Trace:
[ 9306.089506]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9306.096822]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9306.102892]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9306.109351]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9306.115878]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9306.122992]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9306.129861]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9306.136377]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9306.143136]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9306.150330]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9306.157091]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9306.163584]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9306.170598]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9306.177265]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9306.184188]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9306.191393]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9306.198936]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9306.205526]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9306.212884]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9306.221011]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9306.227738]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9306.233062]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9306.239944]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9306.245376] LustreError: dumping log to /tmp/lustre-log.1562681268.40855
[ 9363.280510] Pid: 63214, comm: mdt05_039 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9363.291189] Call Trace:
[ 9363.294450]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9363.301467]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9363.307512]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9363.313943]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9363.320452]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9363.327551]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9363.334398]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9363.340899]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9363.347651]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9363.354827]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9363.361573]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9363.368047]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9363.375042]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9363.381690]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9363.388599]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9363.395792]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9363.403313]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9363.409867]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9363.417180]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9363.425266]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9363.431956]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9363.437240]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9363.444077]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9363.449458] LustreError: dumping log to /tmp/lustre-log.1562681325.63214
[ 9420.485711] Pid: 40887, comm: mdt03_014 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9420.496238] Call Trace:
[ 9420.499917]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9420.506795]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9420.512890]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9420.519262]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9420.525712]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9420.532751]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9420.539545]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9420.545984]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9420.552678]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9420.559801]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9420.566493]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9420.572925]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9420.579870]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9420.586465]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9420.593322]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9420.600481]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9420.607975]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9420.614511]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9420.621819]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9420.629901]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9420.636592]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9420.641868]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9420.648708]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9420.654086] LustreError: dumping log to /tmp/lustre-log.1562681383.40887
[ 9440.916131] Pid: 63196, comm: mdt00_022 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9440.926657] Call Trace:
[ 9440.929988]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[ 9440.937807]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 9440.945391]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 9440.952619]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 9440.960010]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 9440.967308]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 9440.974267]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9440.981132]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9440.988284]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9440.995769]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9441.002314]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9441.009639]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9441.017738]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9441.024449]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9441.029755]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9441.036620]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9441.042034] LustreError: dumping log to /tmp/lustre-log.1562681403.63196
[ 9441.049278] LNet: Service thread pid 63371 was inactive for 200.71s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[ 9540.086373] LustreError: 46542:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562681202, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde74ee0000/0xd5b51f242093cb38 lrc: 3/1,0 mode: --/PR res: [0x14000147cb:0x1a7:0x0].0x0 bits 0x13/0x0 rrc: 30 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 46542 timeout: 0 lvb_type: 0
[ 9540.131713] LustreError: 46542:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 23 previous similar messages
[ 9578.889163] Lustre: f2-MDT0003: Client 3a23b42a-797a-a5a4-a10b-b22688279e6d (at 16@gni5) reconnecting
[ 9578.900007] Lustre: Skipped 140 previous similar messages
[ 9578.907030] Lustre: f2-MDT0003: Connection restored to 3a23b42a-797a-a5a4-a10b-b22688279e6d (at 16@gni5)
[ 9578.917203] Lustre: Skipped 144 previous similar messages
[ 9645.228036] LNet: Service thread pid 63770 was inactive for 1201.47s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[ 9645.248493] LNet: Skipped 9 previous similar messages
[ 9645.254282] Pid: 63770, comm: mdt04_040 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9645.264032] Call Trace:
[ 9645.267041]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9645.273792]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9645.279940]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9645.286466]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9645.293072]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9645.300257]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9645.307199]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9645.313782]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9645.320616]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9645.327881]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9645.334710]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9645.341280]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9645.348361]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9645.355087]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9645.362066]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9645.369335]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9645.376956]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9645.383620]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9645.391054]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9645.399270]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9645.406099]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9645.411519]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9645.418497]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9645.424023] LustreError: dumping log to /tmp/lustre-log.1562681608.63770
[ 9651.357483] Pid: 63807, comm: mdt03_047 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9651.368453] Call Trace:
[ 9651.371867]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[ 9651.379760]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[ 9651.387470]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[ 9651.394820]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[ 9651.402342]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[ 9651.409768]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[ 9651.416841]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9651.423809]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9651.431038]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9651.438601]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9651.445208]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9651.452582]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9651.460727]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9651.467472]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9651.472813]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9651.479704]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9651.485135] LustreError: dumping log to /tmp/lustre-log.1562681614.63807
[ 9755.555542] Pid: 63365, comm: mdt04_029 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9755.566146] Call Trace:
[ 9755.569984]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9755.576782]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9755.582784]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9755.589165]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9755.595624]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9755.602670]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9755.609466]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9755.615920]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9755.622626]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[ 9755.629756]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[ 9755.636448]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9755.642880]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9755.649833]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9755.656439]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9755.663302]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9755.670470]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9755.677987]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9755.684545]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9755.691879]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9755.699978]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9755.706698]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9755.712006]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9755.718870]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9755.724284] LustreError: dumping log to /tmp/lustre-log.1562681719.63365
[ 9825.020787] Pid: 40878, comm: mdt01_016 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9825.031392] Call Trace:
[ 9825.035239]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9825.042031]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9825.048031]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9825.054412]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9825.060863]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9825.067897]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9825.074694]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9825.081135]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9825.087829]  [<ffffffffc19fb05d>] mdt_reint_open+0x1acd/0x27d0 [mdt]
[ 9825.094606]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9825.101039]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9825.107988]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9825.114585]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9825.121444]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9825.128601]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9825.136098]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9825.142629]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9825.149939]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9825.158026]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9825.164734]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9825.170024]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9825.176864]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9825.182250] LustreError: dumping log to /tmp/lustre-log.1562681788.40878
[ 9923.089031] Pid: 203859, comm: mdt04_067 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[ 9923.099637] Call Trace:
[ 9923.102727]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[ 9923.109562]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[ 9923.115554]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[ 9923.121926]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[ 9923.128379]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[ 9923.135420]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[ 9923.142208]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[ 9923.148651]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[ 9923.155342]  [<ffffffffc19fb05d>] mdt_reint_open+0x1acd/0x27d0 [mdt]
[ 9923.162110]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[ 9923.168544]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[ 9923.175494]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[ 9923.182091]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[ 9923.188948]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[ 9923.196106]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[ 9923.203591]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[ 9923.210137]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[ 9923.217452]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[ 9923.225546]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[ 9923.232236]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[ 9923.237526]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[ 9923.244373]  [<ffffffffffffffff>] 0xffffffffffffffff
[ 9923.249753] LustreError: dumping log to /tmp/lustre-log.1562681887.203859
[10045.673812] Pid: 40882, comm: mdt05_024 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10045.684331] Call Trace:
[10045.687472]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[10045.694367]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[10045.700359]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[10045.706731]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[10045.713181]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[10045.720217]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[10045.727014]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[10045.733454]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[10045.740146]  [<ffffffffc19fb05d>] mdt_reint_open+0x1acd/0x27d0 [mdt]
[10045.746926]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[10045.753348]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[10045.760301]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[10045.766896]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10045.773754]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10045.780912]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10045.788407]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10045.794942]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10045.802250]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10045.810333]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10045.817033]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10045.822323]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10045.829173]  [<ffffffffffffffff>] 0xffffffffffffffff
[10045.834553] LustreError: dumping log to /tmp/lustre-log.1562682009.40882
[10080.406262] Lustre: 40849:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-30), not sending early reply
  req@ffff9fde0529e780 x1637721503601744/t0(0) o101->30dfd750-cdf7-d562-e28c-e616526d810f@10.10.32.132@o2ib2:414/0 lens 880/3288 e 0 to 0 dl 1562682049 ref 2 fl Interpret:/0/0 rc 0/0
[10080.440791] Lustre: 40849:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 38 previous similar messages
[10194.675805] Lustre: f2-MDT0003: Client 2eb4c733-9b56-6603-776e-582874f4e830 (at 1004@gni4) reconnecting
[10194.686676] Lustre: Skipped 161 previous similar messages
[10194.692728] Lustre: f2-MDT0003: Connection restored to 2eb4c733-9b56-6603-776e-582874f4e830 (at 1004@gni4)
[10194.703022] Lustre: Skipped 161 previous similar messages
[10200.947105] Pid: 40910, comm: mdt02_014 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10200.957921] Call Trace:
[10200.961965]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10200.969599]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10200.977281]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10200.984601]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10200.992078]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10200.999462]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10201.006500]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10201.013443]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10201.020661]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10201.028225]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10201.034850]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10201.042249]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10201.050413]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10201.057180]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10201.062551]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10201.069477]  [<ffffffffffffffff>] 0xffffffffffffffff
[10201.074951] LustreError: dumping log to /tmp/lustre-log.1562682165.40910
[10201.082942] Pid: 203885, comm: mdt02_073 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10201.093850] Call Trace:
[10201.097319]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10201.105132]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10201.112792]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10201.120096]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10201.127576]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10201.134968]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10201.142014]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10201.148972]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10201.156204]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10201.163782]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10201.170416]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10201.177825]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10201.186012]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10201.192806]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10201.198210]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10201.205174]  [<ffffffffffffffff>] 0xffffffffffffffff
[10201.210680] Pid: 63792, comm: mdt02_041 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10201.220393] Call Trace:
[10201.223349]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10201.230744]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10201.238387]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10201.245669]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10201.253112]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10201.260460]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10201.267456]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10201.274356]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10201.281514]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10201.289016]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10201.295563]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10201.302885]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10201.310985]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10201.317703]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10201.323021]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10201.329899]  [<ffffffffffffffff>] 0xffffffffffffffff
[10352.133619] LNet: Service thread pid 46551 was inactive for 1113.98s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[10352.153516] LNet: Skipped 8 previous similar messages
[10352.159011] Pid: 46551, comm: mdt01_030 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10352.168671] Call Trace:
[10352.171585]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10352.178980]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10352.186621]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10352.193903]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10352.201346]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10352.208706]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10352.215715]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10352.222622]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10352.229798]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10352.237315]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10352.243891]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10352.251242]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10352.259368]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10352.266105]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10352.271457]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10352.278362]  [<ffffffffffffffff>] 0xffffffffffffffff
[10352.283813] LustreError: dumping log to /tmp/lustre-log.1562682317.46551
[10352.291608] Pid: 63758, comm: mdt01_059 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10352.302317] Call Trace:
[10352.305836]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10352.313394]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10352.321022]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10352.328293]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10352.335726]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10352.343075]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10352.350071]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10352.356967]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10352.364136]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10352.371649]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10352.378221]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10352.385569]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10352.393697]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10352.400440]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10352.405784]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10352.412682]  [<ffffffffffffffff>] 0xffffffffffffffff
[10352.418131] Pid: 40862, comm: mdt01_012 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10352.427788] Call Trace:
[10352.430698]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[10352.437357]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[10352.443402]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[10352.449833]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[10352.456334]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[10352.463424]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[10352.470429]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[10352.476971]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[10352.483508]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[10352.489862]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[10352.496651]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[10352.503075]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[10352.510017]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[10352.516612]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10352.523469]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10352.530596]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10352.538072]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10352.544600]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10352.551904]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10352.559989]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10352.566688]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10352.571988]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10352.578844]  [<ffffffffffffffff>] 0xffffffffffffffff
[10352.584233] Pid: 63729, comm: mdt01_052 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10352.593837] Call Trace:
[10352.596703]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10352.604021]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10352.611595]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10352.618806]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10352.626181]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10352.633469]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10352.640413]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10352.647258]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10352.654384]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10352.661854]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10352.668382]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10352.675685]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10352.683772]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10352.690480]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10352.695771]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10352.702627]  [<ffffffffffffffff>] 0xffffffffffffffff
[10352.708016] Pid: 63753, comm: mdt01_058 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10352.717618] Call Trace:
[10352.720477]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[10352.727795]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10352.735370]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10352.742578]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10352.749954]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10352.757241]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10352.764184]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10352.771039]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10352.778166]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10352.785636]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10352.792163]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10352.799476]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10352.807567]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10352.814279]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10352.819595]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10352.826467]  [<ffffffffffffffff>] 0xffffffffffffffff
[10352.831874] LNet: Service thread pid 39022 was inactive for 1114.68s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[10352.845664] LNet: Skipped 5 previous similar messages
[10408.937228] LustreError: 203856:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562682073, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde244dcec0/0xd5b51f242094b0a2 lrc: 3/1,0 mode: --/PR res: [0x14000185bc:0x46d:0x0].0x0 bits 0x13/0x0 rrc: 30 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 203856 timeout: 0 lvb_type: 0
[10408.983081] LustreError: 203856:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 2 previous similar messages
[10409.449900] LustreError: 63763:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562682074, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde246e9f80/0xd5b51f242094b2d2 lrc: 3/1,0 mode: --/PR res: [0x14000185bc:0x46d:0x0].0x0 bits 0x13/0x0 rrc: 30 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 63763 timeout: 0 lvb_type: 0
[10409.496205] LustreError: 63763:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 21 previous similar messages
[10425.683611] LNet: Service thread pid 64059 was inactive for 1202.63s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[10425.700168] LustreError: dumping log to /tmp/lustre-log.1562682390.64059
[10442.028000] LustreError: dumping log to /tmp/lustre-log.1562682407.63233
[10558.244515] LustreError: 39021:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562682223, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde2455cec0/0xd5b51f242094b84a lrc: 3/0,1 mode: --/CW res: [0x1400016e3f:0xa22:0x0].0x0 bits 0x2/0x0 rrc: 32 type: IBT flags: 0x40210400000020 nid: local remote: 0x0 expref: -99 pid: 39021 timeout: 0 lvb_type: 0
[10558.291582] LustreError: 39021:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 25 previous similar messages
[10650.419110] LNet: Service thread pid 40816 was inactive for 1201.20s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[10650.436172] LNet: Skipped 11 previous similar messages
[10650.442197] LustreError: dumping log to /tmp/lustre-log.1562682615.40816
[10702.516835] Lustre: 203858:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdf0f366c00 x1637713107588784/t0(0) o101->fb90bb38-1050-947f-daf2-2169b68871c6@445@gni5:283/0 lens 696/3264 e 4 to 0 dl 1562682673 ref 2 fl Interpret:/0/0 rc 0/0
[10702.553657] Lustre: 203858:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message
[10798.539977] Pid: 63755, comm: mdt05_061 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10798.551442] Call Trace:
[10798.555247]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[10798.563551]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10798.571439]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10798.578959]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10798.586639]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10798.594234]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10798.601472]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10798.608625]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10798.616047]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10798.623813]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10798.630645]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10798.638259]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10798.646630]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10798.653624]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10798.659215]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10798.666350]  [<ffffffffffffffff>] 0xffffffffffffffff
[10798.672024] LustreError: dumping log to /tmp/lustre-log.1562682764.63755
[10798.679824] Pid: 46541, comm: mdt01_025 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10798.689703] Call Trace:
[10798.692819]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[10798.700378]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10798.708185]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10798.715623]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10798.723222]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10798.730724]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10798.737866]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10798.744914]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10798.752231]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10798.759870]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10798.766554]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10798.774005]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10798.782246]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10798.789092]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10798.794525]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10798.801508]  [<ffffffffffffffff>] 0xffffffffffffffff
[10798.807018] Pid: 46562, comm: mdt01_035 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10798.816727] Call Trace:
[10798.819680]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[10798.827074]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10798.834726]  [<ffffffffc19d0458>] mdt_object_local_lock+0x438/0xb20 [mdt]
[10798.842023]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10798.849486]  [<ffffffffc19d0ec0>] mdt_object_lock+0x20/0x30 [mdt]
[10798.856071]  [<ffffffffc19fa1e8>] mdt_reint_open+0xc58/0x27d0 [mdt]
[10798.862814]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[10798.869283]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[10798.876259]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[10798.882883]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10798.889754]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10798.896887]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10798.904368]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10798.910902]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10798.918218]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10798.926313]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10798.933028]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10798.938337]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10798.945208]  [<ffffffffffffffff>] 0xffffffffffffffff
[10798.950615] Pid: 63728, comm: mdt01_051 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10798.960235] Call Trace:
[10798.963110]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[10798.970447]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10798.978039]  [<ffffffffc19d0458>] mdt_object_local_lock+0x438/0xb20 [mdt]
[10798.985276]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10798.992684]  [<ffffffffc19d0ec0>] mdt_object_lock+0x20/0x30 [mdt]
[10798.999221]  [<ffffffffc19fa1e8>] mdt_reint_open+0xc58/0x27d0 [mdt]
[10799.005931]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[10799.012378]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[10799.019329]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[10799.025924]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10799.032783]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10799.039906]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10799.047375]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10799.053907]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10799.061219]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10799.069302]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10799.076003]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10799.081301]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10799.088158]  [<ffffffffffffffff>] 0xffffffffffffffff
[10799.093555] Pid: 46544, comm: mdt01_027 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[10799.103169] Call Trace:
[10799.106033]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[10799.113353]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[10799.120935]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[10799.128163]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[10799.135554]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[10799.142857]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[10799.149811]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[10799.156667]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[10799.163792]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[10799.171261]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[10799.177788]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[10799.185094]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[10799.193186]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[10799.199897]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[10799.205205]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[10799.212068]  [<ffffffffffffffff>] 0xffffffffffffffff
[10799.217467] LNet: Service thread pid 203834 was inactive for 201.42s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[10799.231248] LNet: Skipped 1 previous similar message
[10800.474212] Lustre: f2-MDT0003: Client 0c0d79b5-17cb-ead7-92e3-17f5534ca5ac (at 967@gni4) reconnecting
[10800.484983] Lustre: Skipped 188 previous similar messages
[10800.491865] Lustre: f2-MDT0003: Connection restored to 0c0d79b5-17cb-ead7-92e3-17f5534ca5ac (at 967@gni4)
[10800.502066] Lustre: Skipped 188 previous similar messages
[10822.034992] LustreError: dumping log to /tmp/lustre-log.1562682787.203843
[10897.556793] LustreError: 63759:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562682563, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde547486c0/0xd5b51f24209502fe lrc: 3/1,0 mode: --/PR res: [0x14000146fb:0x274:0x0].0x0 bits 0x13/0x0 rrc: 34 type: IBT flags: 0x40210400000020 nid: local remote: 0x0 expref: -99 pid: 63759 timeout: 0 lvb_type: 0
[10897.602494] LustreError: 63759:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 19 previous similar messages
[10948.703762] LustreError: dumping log to /tmp/lustre-log.1562682914.203880
[11308.278824] LNet: Service thread pid 40896 was inactive for 1201.03s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[11308.299307] LNet: Skipped 9 previous similar messages
[11308.305086] Pid: 40896, comm: mdt05_027 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11308.314837] Call Trace:
[11308.317842]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11308.325339]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11308.333084]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11308.340459]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11308.348009]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11308.355460]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11308.362568]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11308.369581]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11308.376861]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11308.384493]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11308.391185]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11308.398645]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11308.406881]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11308.413727]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11308.419171]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11308.426166]  [<ffffffffffffffff>] 0xffffffffffffffff
[11308.431709] LustreError: dumping log to /tmp/lustre-log.1562683275.40896
[11308.440322] Pid: 40894, comm: mdt00_019 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11308.451357] Call Trace:
[11308.455380]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11308.462927]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11308.470697]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11308.478065]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11308.485608]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11308.493066]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11308.500165]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11308.507175]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11308.514461]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11308.522099]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11308.528787]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11308.536234]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11308.544457]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11308.551288]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11308.556707]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11308.563676]  [<ffffffffffffffff>] 0xffffffffffffffff
[11308.569189] Pid: 203872, comm: mdt00_075 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11308.578989] Call Trace:
[11308.581949]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11308.589354]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11308.597005]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11308.604295]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11308.611738]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11308.619087]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11308.626082]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11308.632984]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11308.640147]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11308.647653]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11308.654197]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11308.661525]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11308.669630]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11308.676348]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11308.681670]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11308.688547]  [<ffffffffffffffff>] 0xffffffffffffffff
[11308.693952] Pid: 203829, comm: mdt00_061 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11308.703674] Call Trace:
[11308.706564]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11308.713904]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11308.721506]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11308.728751]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11308.736169]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11308.743490]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11308.750468]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11308.757359]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11308.764523]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11308.772029]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11308.778586]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11308.785906]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11308.794010]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11308.800725]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11308.806034]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11308.812909]  [<ffffffffffffffff>] 0xffffffffffffffff
[11308.818323] Pid: 63744, comm: mdt00_037 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11308.827953] Call Trace:
[11308.830836]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11308.838180]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11308.845782]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11308.853033]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11308.860450]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11308.867773]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11308.874751]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11308.881641]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11308.888799]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11308.896304]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11308.902857]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11308.910180]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11308.918281]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11308.925001]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11308.930317]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11308.937190]  [<ffffffffffffffff>] 0xffffffffffffffff
[11308.942596] LNet: Service thread pid 203856 was inactive for 1202.18s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[11308.956472] LNet: Skipped 14 previous similar messages
[11345.053557] LustreError: dumping log to /tmp/lustre-log.1562683312.46536
[11403.293459] Lustre: f2-MDT0003: Client e47ef03d-c378-b5d3-0bb5-4b46733e4a9b (at 1017@gni4) reconnecting
[11403.304349] Lustre: Skipped 238 previous similar messages
[11403.311271] Lustre: f2-MDT0003: Connection restored to e47ef03d-c378-b5d3-0bb5-4b46733e4a9b (at 1017@gni4)
[11403.321559] Lustre: Skipped 237 previous similar messages
[11407.877027] Lustre: 40852:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcf807a880 x1637721134891472/t0(0) o101->d4b09ee4-18e0-b7aa-bc49-f946781e9a4a@10.10.32.133@o2ib2:235/0 lens 912/3288 e 1 to 0 dl 1562683380 ref 2 fl Interpret:/0/0 rc 0/0
[11407.912432] Lustre: 40852:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 80 previous similar messages
[11434.947335] LustreError: dumping log to /tmp/lustre-log.1562683402.40837
[11447.205586] LustreError: dumping log to /tmp/lustre-log.1562683414.109288
[11459.463719] LustreError: dumping log to /tmp/lustre-log.1562683426.40821
[11775.808076] LustreError: 63266:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562683444, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde59c4c800/0xd5b51f242097cfc1 lrc: 3/1,0 mode: --/PR res: [0x1400014592:0x12aff:0x0].0x0 bits 0x13/0x0 rrc: 9 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 63266 timeout: 0 lvb_type: 0
[11798.607618] Pid: 40814, comm: mdt04_004 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11798.618681] Call Trace:
[11798.622373]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11798.630230]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11798.637990]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11798.645375]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11798.652921]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11798.660379]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11798.667491]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11798.674511]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11798.681791]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11798.689417]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11798.696098]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11798.703559]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11798.711793]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11798.718649]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11798.724092]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11798.731097]  [<ffffffffffffffff>] 0xffffffffffffffff
[11798.736649] LustreError: dumping log to /tmp/lustre-log.1562683767.40814
[11798.745293] Pid: 63863, comm: mdt04_059 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11798.756356] Call Trace:
[11798.759592]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11798.767296]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11798.775026]  [<ffffffffc19d0458>] mdt_object_local_lock+0x438/0xb20 [mdt]
[11798.782399]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11798.789947]  [<ffffffffc19d0ec0>] mdt_object_lock+0x20/0x30 [mdt]
[11798.796624]  [<ffffffffc19fa1e8>] mdt_reint_open+0xc58/0x27d0 [mdt]
[11798.803478]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[11798.810058]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[11798.817139]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[11798.823873]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11798.830859]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11798.838115]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11798.845706]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11798.852347]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11798.859757]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11798.867942]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11798.874719]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11798.880087]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11798.887004]  [<ffffffffffffffff>] 0xffffffffffffffff
[11798.892456] Pid: 46533, comm: mdt04_019 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11798.902114] Call Trace:
[11798.905030]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[11798.911698]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[11798.917750]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[11798.924174]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[11798.930656]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[11798.937711]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[11798.944692]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[11798.951211]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[11798.957736]  [<ffffffffc1b2ba50>] mdd_create+0xac0/0x14b0 [mdd]
[11798.964073]  [<ffffffffc19faf60>] mdt_reint_open+0x19d0/0x27d0 [mdt]
[11798.970844]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[11798.977268]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[11798.984202]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[11798.990788]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11798.997644]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11799.004761]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11799.012230]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11799.018760]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11799.026062]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11799.034142]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11799.040840]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11799.046120]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11799.052968]  [<ffffffffffffffff>] 0xffffffffffffffff
[11799.058350] Pid: 203874, comm: mdt04_071 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[11799.068035] Call Trace:
[11799.070877]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[11799.078180]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[11799.085727]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[11799.092921]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[11799.100286]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[11799.107566]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[11799.114502]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[11799.121349]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[11799.128465]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[11799.135933]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[11799.142453]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[11799.149759]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[11799.157843]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[11799.164542]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[11799.169836]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[11799.176689]  [<ffffffffffffffff>] 0xffffffffffffffff
[11974.237658] LustreError: 39026:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562683642, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fdf5f505340/0xd5b51f242097f45e lrc: 3/1,0 mode: --/PR res: [0x14000147ad:0x4395:0x0].0x0 bits 0x13/0x0 rrc: 31 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 39026 timeout: 0 lvb_type: 0
[11974.282808] LustreError: 39026:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 22 previous similar messages
[12003.476892] Lustre: f2-MDT0003: Client 7c8db4c2-2490-4adc-0e03-61b42292820c (at 528@gni5) reconnecting
[12003.487675] Lustre: Skipped 149 previous similar messages
[12003.493799] Lustre: f2-MDT0003: Connection restored to 7c8db4c2-2490-4adc-0e03-61b42292820c (at 528@gni5)
[12003.504076] Lustre: Skipped 150 previous similar messages
[12008.018535] LNet: Service thread pid 109299 was inactive for 200.70s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[12008.038750] LNet: Skipped 8 previous similar messages
[12008.044506] Pid: 109299, comm: mdt_rdpg03_019 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12008.054734] Call Trace:
[12008.057707]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[12008.064433]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[12008.070545]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[12008.077038]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[12008.083609]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[12008.090766]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[12008.097684]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[12008.103988]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12008.111456]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12008.119687]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12008.126508]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12008.131940]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12008.138925]  [<ffffffffffffffff>] 0xffffffffffffffff
[12008.144449] LustreError: dumping log to /tmp/lustre-log.1562683976.109299
[12015.169163] LNet: Service thread pid 63760 was inactive for 1203.32s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[12015.185626] LNet: Skipped 53 previous similar messages
[12015.191532] LustreError: dumping log to /tmp/lustre-log.1562683984.63760
[12038.131394] Lustre: 109289:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcb9f28480 x1637720428265744/t0(0) o101->37723e5a-ef97-8aea-4a01-ee00091bb44c@10.10.32.135@o2ib2:112/0 lens 880/3288 e 1 to 0 dl 1562684012 ref 2 fl Interpret:/0/0 rc 0/0
[12411.517806] Pid: 203869, comm: mdt04_069 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12411.529051] Call Trace:
[12411.533456]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[12411.540499]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[12411.546759]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[12411.553332]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[12411.559980]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[12411.567206]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[12411.574194]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[12411.580824]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[12411.587704]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[12411.595021]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[12411.601884]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[12411.608473]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[12411.615570]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[12411.622307]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12411.629291]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12411.636572]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12411.644179]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12411.650827]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12411.658250]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12411.666434]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12411.673212]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12411.678591]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12411.685519]  [<ffffffffffffffff>] 0xffffffffffffffff
[12411.690992] LustreError: dumping log to /tmp/lustre-log.1562684381.203869
[12610.114032] Lustre: f2-MDT0003: Client 37723e5a-ef97-8aea-4a01-ee00091bb44c (at 10.10.32.135@o2ib2) reconnecting
[12610.125767] Lustre: Skipped 270 previous similar messages
[12610.132352] Lustre: f2-MDT0003: Connection restored to 37723e5a-ef97-8aea-4a01-ee00091bb44c (at 10.10.32.135@o2ib2)
[12610.143262] Lustre: Skipped 269 previous similar messages
[12644.424045] LNet: Service thread pid 40839 was inactive for 1201.82s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[12644.444417] LNet: Skipped 1 previous similar message
[12644.449901] Pid: 40839, comm: mdt03_008 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12644.459636] Call Trace:
[12644.462623]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[12644.469358]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[12644.475490]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[12644.481989]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[12644.488571]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[12644.495719]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[12644.502609]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[12644.509156]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[12644.515942]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[12644.523152]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[12644.529914]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[12644.536405]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[12644.543400]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[12644.550040]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12644.556932]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12644.564125]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12644.571656]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12644.578227]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12644.585574]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12644.593703]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12644.600437]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12644.605771]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12644.612670]  [<ffffffffffffffff>] 0xffffffffffffffff
[12644.618110] LustreError: dumping log to /tmp/lustre-log.1562684614.40839
[12677.112666] Pid: 40852, comm: mdt01_008 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12677.123374] Call Trace:
[12677.127120]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[12677.133968]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[12677.140019]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[12677.146451]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[12677.152963]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[12677.160062]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[12677.166906]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[12677.173400]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[12677.180143]  [<ffffffffc19fb05d>] mdt_reint_open+0x1acd/0x27d0 [mdt]
[12677.186975]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[12677.193460]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[12677.200453]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[12677.207103]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12677.214002]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12677.221188]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12677.228698]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12677.235253]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12677.242571]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12677.250661]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12677.257358]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12677.262642]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12677.269481]  [<ffffffffffffffff>] 0xffffffffffffffff
[12677.274862] LustreError: dumping log to /tmp/lustre-log.1562684647.40852
[12677.282130] Pid: 63266, comm: mdt01_046 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12677.291729] Call Trace:
[12677.294576]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[12677.301886]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[12677.309451]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[12677.316661]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[12677.324044]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[12677.331341]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[12677.338294]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12677.345157]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12677.352290]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12677.359769]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12677.366304]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12677.373619]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12677.381712]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12677.388424]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12677.393739]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12677.400604]  [<ffffffffffffffff>] 0xffffffffffffffff
[12738.403784] Pid: 40848, comm: mdt03_009 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12738.414382] Call Trace:
[12738.418031]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[12738.424832]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[12738.430850]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[12738.437249]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[12738.443712]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[12738.450751]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[12738.457545]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[12738.463989]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[12738.470678]  [<ffffffffc19f9252>] mdt_open_by_fid_lock+0x672/0x9b0 [mdt]
[12738.477803]  [<ffffffffc19f9cf0>] mdt_reint_open+0x760/0x27d0 [mdt]
[12738.484496]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[12738.490926]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[12738.497871]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[12738.504476]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12738.511340]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12738.518511]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12738.526024]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12738.532574]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12738.539899]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12738.547998]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12738.554706]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12738.560014]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12738.566879]  [<ffffffffffffffff>] 0xffffffffffffffff
[12738.572295] LustreError: dumping log to /tmp/lustre-log.1562684709.40848
[12824.211295] Pid: 63727, comm: mdt05_057 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12824.221894] Call Trace:
[12824.225531]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[12824.232349]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[12824.238361]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[12824.244748]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[12824.251199]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[12824.258246]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[12824.265041]  [<ffffffffc19f7eee>] mdt_mfd_open+0x8de/0xe70 [mdt]
[12824.271491]  [<ffffffffc19f8acb>] mdt_finish_open+0x64b/0x760 [mdt]
[12824.278191]  [<ffffffffc19fb05d>] mdt_reint_open+0x1acd/0x27d0 [mdt]
[12824.284970]  [<ffffffffc19edfa3>] mdt_reint_rec+0x83/0x210 [mdt]
[12824.291401]  [<ffffffffc19cc1b3>] mdt_reint_internal+0x6e3/0xaf0 [mdt]
[12824.298354]  [<ffffffffc19d8972>] mdt_intent_open+0x82/0x3a0 [mdt]
[12824.304958]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12824.311822]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12824.318984]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12824.326495]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12824.333032]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12824.340347]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12824.348429]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12824.355130]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12824.360421]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12824.367267]  [<ffffffffffffffff>] 0xffffffffffffffff
[12824.372658] LustreError: dumping log to /tmp/lustre-log.1562684795.63727
[12873.244165] Pid: 40881, comm: mdt02_007 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12873.254673] Call Trace:
[12873.258116]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[12873.266036]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[12873.273620]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[12873.280845]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[12873.288238]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[12873.295536]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[12873.302486]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12873.309353]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12873.316493]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12873.323971]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12873.330517]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12873.337839]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12873.345931]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12873.352641]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12873.357948]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12873.364816]  [<ffffffffffffffff>] 0xffffffffffffffff
[12873.370229] LustreError: dumping log to /tmp/lustre-log.1562684844.40881
[12873.377704] Pid: 203837, comm: mdt03_064 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12873.377705] Call Trace:
[12873.377761]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[12873.377807]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[12873.377826]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[12873.377849]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[12873.377867]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[12873.377886]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[12873.377903]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12873.377942]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12873.377990]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12873.378061]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12873.378130]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12873.378188]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12873.378244]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12873.378248]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12873.378255]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12873.378274]  [<ffffffffffffffff>] 0xffffffffffffffff
[12873.378280] Pid: 63865, comm: mdt02_059 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[12873.378281] Call Trace:
[12873.378331]  [<ffffffffc1696d9d>] ldlm_completion_ast+0x63d/0x920 [ptlrpc]
[12873.378375]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[12873.378392]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[12873.378409]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[12873.378427]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[12873.378444]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[12873.378461]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[12873.378500]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[12873.378546]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[12873.378614]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[12873.378680]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[12873.378736]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[12873.378792]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[12873.378795]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[12873.378799]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[12873.378805]  [<ffffffffffffffff>] 0xffffffffffffffff
[12873.378811] LNet: Service thread pid 46522 was inactive for 1201.28s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[13190.935869] Pid: 63775, comm: mdt05_065 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[13190.946530] Call Trace:
[13190.949746]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[13190.957465]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[13190.965099]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[13190.972366]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[13190.979806]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[13190.987147]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[13190.994142]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[13191.001041]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[13191.008215]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[13191.015719]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[13191.022285]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[13191.029633]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[13191.037752]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[13191.044492]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[13191.049822]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[13191.056719]  [<ffffffffffffffff>] 0xffffffffffffffff
[13191.062159] LustreError: dumping log to /tmp/lustre-log.1562685162.63775
[13191.070600] Pid: 40888, comm: mdt02_008 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[13191.081278] Call Trace:
[13191.084840]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[13191.092210]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[13191.099824]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[13191.107090]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[13191.114523]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[13191.121865]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[13191.128859]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[13191.135758]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[13191.142924]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[13191.150441]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[13191.157016]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[13191.164358]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[13191.172477]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[13191.179220]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[13191.184559]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[13191.191466]  [<ffffffffffffffff>] 0xffffffffffffffff
[13191.196911] Pid: 203886, comm: mdt03_075 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[13191.206653] Call Trace:
[13191.209561]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[13191.216934]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[13191.224559]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[13191.231819]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[13191.239255]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[13191.246605]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[13191.253609]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[13191.260516]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[13191.267675]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[13191.275179]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[13191.281724]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[13191.289046]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[13191.297152]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[13191.303874]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[13191.309191]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[13191.316063]  [<ffffffffffffffff>] 0xffffffffffffffff
[13191.321487] Pid: 109289, comm: mdt03_057 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[13191.331207] Call Trace:
[13191.334097]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[13191.341440]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[13191.349043]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[13191.356294]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[13191.363710]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[13191.371037]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[13191.378027]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[13191.384919]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[13191.392079]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[13191.399583]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[13191.406137]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[13191.413469]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[13191.421569]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[13191.428288]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[13191.433593]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[13191.440467]  [<ffffffffffffffff>] 0xffffffffffffffff
[13191.445875] Pid: 46526, comm: mdt03_020 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[13191.455514] Call Trace:
[13191.458399]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[13191.465741]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[13191.473339]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[13191.480587]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[13191.488002]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[13191.495326]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[13191.502302]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[13191.509197]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[13191.516360]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[13191.523864]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[13191.530419]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[13191.537741]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[13191.545846]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[13191.552569]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[13191.557877]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[13191.564752]  [<ffffffffffffffff>] 0xffffffffffffffff
[13215.546648] Lustre: f2-MDT0003: Client 7834f4e5-191f-40f1-86bb-4a218ecbe791 (at 502@gni5) reconnecting
[13215.557372] Lustre: Skipped 269 previous similar messages
[13215.563736] Lustre: f2-MDT0003: Connection restored to 7834f4e5-191f-40f1-86bb-4a218ecbe791 (at 502@gni5)
[13215.573872] Lustre: Skipped 269 previous similar messages
[13290.379066] LustreError: 203830:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562684962, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fdf50055a00/0xd5b51f242098a835 lrc: 3/1,0 mode: --/PR res: [0x14000184b4:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 33 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 203830 timeout: 0 lvb_type: 0
[13290.424915] LustreError: 203830:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 21 previous similar messages
[13584.602127] Lustre: 39023:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fde78cb2400 x1637713649683664/t0(0) o101->f1b85999-07ea-798a-28ad-99b5ca8c1b8f@973@gni5:152/0 lens 696/3264 e 22 to 0 dl 1562685562 ref 2 fl Interpret:/0/0 rc 0/0
[13584.637089] Lustre: 39023:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 38 previous similar messages
[13832.803296] Lustre: f2-MDT0003: Client 3072eeeb-8262-275f-9798-4695f00d1bda (at 85@gni4) reconnecting
[13832.814294] Lustre: Skipped 297 previous similar messages
[13832.820496] Lustre: f2-MDT0003: Connection restored to 3072eeeb-8262-275f-9798-4695f00d1bda (at 85@gni4)
[13832.830773] Lustre: Skipped 297 previous similar messages
[14382.531503] LNet: Service thread pid 63784 was inactive for 200.38s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[14382.552218] LNet: Skipped 12 previous similar messages
[14382.557945] Pid: 63784, comm: mdt01_065 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[14382.567740] Call Trace:
[14382.570798]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[14382.578338]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[14382.586121]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[14382.593538]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[14382.601132]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[14382.608638]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[14382.615796]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[14382.622847]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[14382.630165]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[14382.637834]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[14382.644559]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[14382.652056]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[14382.660340]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[14382.667239]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[14382.672727]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[14382.679781]  [<ffffffffffffffff>] 0xffffffffffffffff
[14382.685370] LustreError: dumping log to /tmp/lustre-log.1562686357.63784
[14382.697637] Pid: 39035, comm: mdt05_002 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[14382.708815] Call Trace:
[14382.712105]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[14382.719852]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[14382.727617]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[14382.735025]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[14382.742607]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[14382.750106]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[14382.757246]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[14382.764296]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[14382.771605]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[14382.779256]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[14382.785958]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[14382.793420]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[14382.801653]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[14382.808489]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[14382.813910]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[14382.820878]  [<ffffffffffffffff>] 0xffffffffffffffff
[14382.826380] Pid: 40869, comm: mdt05_022 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[14382.836081] Call Trace:
[14382.839022]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[14382.846410]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[14382.854044]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[14382.861326]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[14382.868776]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[14382.876126]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[14382.883119]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[14382.890016]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[14382.897178]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[14382.904683]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[14382.911229]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[14382.918550]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[14382.926652]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[14382.933368]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[14382.938678]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[14382.945550]  [<ffffffffffffffff>] 0xffffffffffffffff
[14382.950964] Pid: 203887, comm: mdt04_075 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[14382.960681] Call Trace:
[14382.963567]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[14382.970907]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[14382.978508]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[14382.985754]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[14382.993172]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[14383.000498]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[14383.007490]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[14383.014380]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[14383.021538]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[14383.029045]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[14383.035605]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[14383.042927]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[14383.051037]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[14383.057754]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[14383.063062]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[14383.069937]  [<ffffffffffffffff>] 0xffffffffffffffff
[14383.075341] Pid: 63754, comm: mdt04_038 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[14383.084964] Call Trace:
[14383.087847]  [<ffffffffc1696d11>] ldlm_completion_ast+0x5b1/0x920 [ptlrpc]
[14383.095191]  [<ffffffffc1697aac>] ldlm_cli_enqueue_local+0x23c/0x870 [ptlrpc]
[14383.102794]  [<ffffffffc19d052b>] mdt_object_local_lock+0x50b/0xb20 [mdt]
[14383.110047]  [<ffffffffc19d0bb0>] mdt_object_lock_internal+0x70/0x360 [mdt]
[14383.117456]  [<ffffffffc19d1d1a>] mdt_getattr_name_lock+0x90a/0x1c30 [mdt]
[14383.124786]  [<ffffffffc19d9c05>] mdt_intent_getattr+0x2b5/0x480 [mdt]
[14383.131763]  [<ffffffffc19d6a18>] mdt_intent_policy+0x2e8/0xd00 [mdt]
[14383.138655]  [<ffffffffc167dd26>] ldlm_lock_enqueue+0x366/0xa60 [ptlrpc]
[14383.145815]  [<ffffffffc16a6587>] ldlm_handle_enqueue0+0xa47/0x15a0 [ptlrpc]
[14383.153326]  [<ffffffffc172e882>] tgt_enqueue+0x62/0x210 [ptlrpc]
[14383.159878]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[14383.167200]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[14383.175306]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[14383.182027]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[14383.187338]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[14383.194211]  [<ffffffffffffffff>] 0xffffffffffffffff
[14383.199616] LNet: Service thread pid 40831 was inactive for 201.10s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[14383.213323] LNet: Skipped 48 previous similar messages
[14471.381569] Lustre: f2-MDT0003: Client 3072eeeb-8262-275f-9798-4695f00d1bda (at 85@gni4) reconnecting
[14471.392283] Lustre: Skipped 298 previous similar messages
[14471.398390] Lustre: f2-MDT0003: Connection restored to 3072eeeb-8262-275f-9798-4695f00d1bda (at 85@gni4)
[14471.408552] Lustre: Skipped 298 previous similar messages
[14481.899727] LustreError: 46543:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562686156, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fde23dead00/0xd5b51f242098e234 lrc: 3/1,0 mode: --/PR res: [0x1400014b5c:0x536:0x0].0x0 bits 0x13/0x0 rrc: 33 type: IBT flags: 0x40210000000000 nid: local remote: 0x0 expref: -99 pid: 46543 timeout: 0 lvb_type: 0
[14481.945139] LustreError: 46543:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 21 previous similar messages
[14492.344340] LNet: Service thread pid 63852 was inactive for 210.66s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[14492.360591] LNet: Skipped 21 previous similar messages
[14492.366700] LustreError: dumping log to /tmp/lustre-log.1562686467.63852
[14581.412585] LustreError: 63665:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1562686256, 300s ago); not entering recovery in server code, just going back to sleep ns: mdt-f2-MDT0003_UUID lock: ffff9fdf54239f80/0xd5b51f242098e7c1 lrc: 3/0,1 mode: --/PW res: [0x14000147ad:0x4393:0x0].0x65726f63 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210400000020 nid: local remote: 0x0 expref: -99 pid: 63665 timeout: 0 lvb_type: 0
[14581.459873] LustreError: 63665:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 3 previous similar messages
[14775.495257] Lustre: 40850:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fde79f76300 x1637713738869392/t0(0) o101->39dc5a06-84ab-e47b-a77e-5c0e063bc88c@617@gni5:591/0 lens 744/3264 e 22 to 0 dl 1562686756 ref 2 fl Interpret:/0/0 rc 0/0
[14775.531119] Lustre: 40850:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 26 previous similar messages
[14809.437607] LustreError: 420382:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 628s
[14816.467537] Lustre: 63212:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fde1ccb4380 x1637713207635264/t0(0) o101->89af946b-1374-e507-e694-12549319bf48@609@gni5:632/0 lens 376/0 e 11 to 0 dl 1562686797 ref 2 fl New:/0/ffffffff rc 0/-1
[14816.504347] Lustre: 63212:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 24 previous similar messages
[14858.449390] Lustre: 40865:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd24cf2d00 x1637721482002416/t0(0) o41->6b380e87-16ee-2837-6f80-d86a273b02ae@10.10.32.129@o2ib2:674/0 lens 224/0 e 5 to 0 dl 1562686839 ref 2 fl New:/0/ffffffff rc 0/-1
[14858.486858] Lustre: 40865:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1 previous similar message
[14869.587151] LustreError: 420481:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 688s
[14905.546695] Lustre: 63548:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcce6e0900 x1637714971709968/t0(0) o35->3986f32f-4a2b-fc77-2e1f-da67fb2161bb@3675@gni4:721/0 lens 512/456 e 11 to 0 dl 1562686886 ref 2 fl Interpret:/0/0 rc 0/0
[14905.579785] Lustre: 63548:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 403 previous similar messages
[14929.521754] LustreError: 420558:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 648s
[14981.777148] Lustre: 63367:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdcca689680 x1637722237489088/t0(0) o41->e7566399-e686-8c37-be89-39e485b4b52e@10.10.32.233@o2ib2:42/0 lens 224/0 e 1 to 0 dl 1562686962 ref 2 fl New:/0/ffffffff rc 0/-1
[14981.816563] Lustre: 63367:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 16 previous similar messages
[14989.455347] LustreError: 420746:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 708s
[15049.386127] LustreError: 420919:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 769s
[15095.994997] Lustre: f2-MDT0003: Client f6e377e1-2cdf-8de4-4c96-d915bbd84419 (at 1031@gni5) reconnecting
[15096.007145] Lustre: Skipped 451 previous similar messages
[15096.014473] Lustre: f2-MDT0003: Connection restored to f6e377e1-2cdf-8de4-4c96-d915bbd84419 (at 1031@gni5)
[15096.024946] Lustre: Skipped 451 previous similar messages
[15109.320294] LustreError: 420995:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 829s
[15148.283800] Lustre: 133918:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd3f18ec00 x1637713112499232/t0(0) o41->73888169-063b-0913-e479-171059db1719@782@gni5:209/0 lens 224/0 e 1 to 0 dl 1562687129 ref 2 fl New:/0/ffffffff rc 0/-1
[15148.321274] Lustre: 133918:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 20 previous similar messages
[15169.253318] LustreError: 421071:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 889s
[15229.186664] LustreError: 421148:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 949s
[15349.035776] LustreError: 421575:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 1069s
[15349.052492] LustreError: 421575:0:(service.c:3336:ptlrpc_svcpt_health_check()) Skipped 1 previous similar message
[15455.760077] Lustre: 63212:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd6a33ec00 x1637713207635264/t0(0) o101->89af946b-1374-e507-e694-12549319bf48@609@gni5:517/0 lens 376/0 e 0 to 0 dl 1562687437 ref 2 fl New:/2/ffffffff rc 0/-1
[15455.798683] Lustre: 63212:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 11 previous similar messages
[15480.148778] LNet: Service thread pid 109308 was inactive for 1200.91s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[15480.171196] LNet: Skipped 4 previous similar messages
[15480.177055] Pid: 109308, comm: mdt_rdpg01_037 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[15480.187582] Call Trace:
[15480.190841]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[15480.197835]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[15480.204207]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[15480.210942]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[15480.217761]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[15480.225157]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[15480.232290]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[15480.238790]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[15480.246459]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[15480.254880]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[15480.261883]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[15480.267478]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[15480.274618]  [<ffffffffffffffff>] 0xffffffffffffffff
[15480.280292] LustreError: dumping log to /tmp/lustre-log.1562687457.109308
[15480.292693] Pid: 204152, comm: mdt_rdpg03_036 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[15480.304607] Call Trace:
[15480.308419]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[15480.315359]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[15480.321583]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[15480.328164]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[15480.334821]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[15480.342055]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[15480.349042]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[15480.355405]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[15480.362909]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[15480.371184]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[15480.378066]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[15480.383530]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[15480.390543]  [<ffffffffffffffff>] 0xffffffffffffffff
[15480.396087] Pid: 128085, comm: mdt_rdpg03_022 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[15480.406356] Call Trace:
[15480.409352]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[15480.416081]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[15480.422195]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[15480.428669]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[15480.435231]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[15480.442381]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[15480.449263]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[15480.455522]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[15480.462916]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[15480.471075]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[15480.477837]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[15480.483188]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[15480.490089]  [<ffffffffffffffff>] 0xffffffffffffffff
[15480.495510] Pid: 203925, comm: mdt_rdpg03_029 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[15480.505669] Call Trace:
[15480.508560]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[15480.515195]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[15480.521213]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[15480.527594]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[15480.534052]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[15480.541117]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[15480.547919]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[15480.554118]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[15480.561458]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[15480.569578]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[15480.576303]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[15480.581629]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[15480.588511]  [<ffffffffffffffff>] 0xffffffffffffffff
[15480.593927] Pid: 51776, comm: mdt_rdpg03_010 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[15480.603996] Call Trace:
[15480.606887]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[15480.613524]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[15480.619540]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[15480.625914]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[15480.632371]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[15480.639426]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[15480.646231]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[15480.652422]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[15480.659754]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[15480.667863]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[15480.674599]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[15480.679924]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[15480.686804]  [<ffffffffffffffff>] 0xffffffffffffffff
[15480.692221] LNet: Service thread pid 39043 was inactive for 1201.46s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[15480.706037] LNet: Skipped 93 previous similar messages
[15512.837337] LNet: Service thread pid 204161 was inactive for 1203.92s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[15512.853306] LNet: Skipped 103 previous similar messages
[15512.859581] LustreError: dumping log to /tmp/lustre-log.1562687490.204161
[15528.043245] LustreError: 421834:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 1248s
[15528.057763] LustreError: 421834:0:(service.c:3336:ptlrpc_svcpt_health_check()) Skipped 3 previous similar messages
[15563.002894] perf: interrupt took too long (2501 > 2500), lowering kernel.perf_event_max_sample_rate to 79000
[15563.145212] perf: interrupt took too long (3128 > 3126), lowering kernel.perf_event_max_sample_rate to 63000
[15563.340048] perf: interrupt took too long (3931 > 3910), lowering kernel.perf_event_max_sample_rate to 50000
[15563.548828] perf: interrupt took too long (4926 > 4913), lowering kernel.perf_event_max_sample_rate to 40000
[15567.899384] perf: interrupt took too long (6269 > 6157), lowering kernel.perf_event_max_sample_rate to 31000
[15567.911140] perf: interrupt took too long (9879 > 9821), lowering kernel.perf_event_max_sample_rate to 20000
[15697.641522] Lustre: f2-MDT0003: Client ddabc6f0-145f-cc43-452f-9b5e29df903b (at 10.10.32.108@o2ib2) reconnecting
[15697.653492] Lustre: Skipped 451 previous similar messages
[15697.660604] Lustre: f2-MDT0003: Connection restored to ddabc6f0-145f-cc43-452f-9b5e29df903b (at 10.10.32.108@o2ib2)
[15697.671735] Lustre: Skipped 451 previous similar messages
[15827.662296] LustreError: 422550:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 1549s
[15827.677389] LustreError: 422550:0:(service.c:3336:ptlrpc_svcpt_health_check()) Skipped 9 previous similar messages
[16057.089904] Lustre: 63208:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdd28f1ba80 x1637713597301328/t0(0) o101->76a56203-646f-d1f7-2bd7-7e18edbc5bb4@914@gni5:365/0 lens 376/0 e 0 to 0 dl 1562688040 ref 2 fl New:/2/ffffffff rc 0/-1
[16057.125586] Lustre: 63208:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 473 previous similar messages
[16068.542006] LNet: Service thread pid 64046 was inactive for 1203.74s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[16068.563285] LNet: Skipped 4 previous similar messages
[16068.568965] Pid: 64046, comm: mdt_rdpg05_033 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16068.579234] Call Trace:
[16068.582316]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16068.589140]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16068.595348]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16068.601942]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16068.608617]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16068.615872]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[16068.623091]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[16068.629827]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[16068.636560]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[16068.643200]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[16068.649935]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[16068.657094]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[16068.663457]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16068.670998]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16068.679306]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16068.686204]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16068.691678]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16068.698732]  [<ffffffffffffffff>] 0xffffffffffffffff
[16068.704285] LustreError: dumping log to /tmp/lustre-log.1562688047.64046
[16068.713016] Pid: 204110, comm: mdt_rdpg00_030 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16068.723315] Call Trace:
[16068.726336]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16068.733100]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16068.739248]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16068.745749]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16068.752319]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16068.759486]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[16068.766600]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[16068.773258]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[16068.779907]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[16068.786473]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[16068.793129]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[16068.800210]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[16068.806494]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16068.813913]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16068.822110]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16068.828903]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16068.834273]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16068.841205]  [<ffffffffffffffff>] 0xffffffffffffffff
[16068.846648] Pid: 63574, comm: mdt_rdpg05_025 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16068.856743] Call Trace:
[16068.859642]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16068.866278]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16068.872297]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16068.878675]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16068.885135]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16068.892172]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[16068.899136]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[16068.905644]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[16068.912144]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[16068.918561]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[16068.925060]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[16068.932008]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[16068.938171]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16068.945487]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16068.953576]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16068.960276]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16068.965567]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16068.972430]  [<ffffffffffffffff>] 0xffffffffffffffff
[16068.977821] Pid: 64054, comm: mdt_rdpg01_028 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16068.987869] Call Trace:
[16068.990732]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16068.997351]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16069.003367]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16069.009728]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16069.016184]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16069.023228]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[16069.030204]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[16069.036712]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[16069.043207]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[16069.049615]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[16069.056114]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[16069.063051]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[16069.069204]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16069.076502]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16069.084586]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16069.091275]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16069.096568]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16069.103416]  [<ffffffffffffffff>] 0xffffffffffffffff
[16069.108804] Pid: 63594, comm: mdt_rdpg01_022 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16069.118853] Call Trace:
[16069.121713]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16069.128322]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16069.134325]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16069.140694]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16069.147135]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16069.154174]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[16069.161133]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[16069.167635]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[16069.174129]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[16069.180541]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[16069.187046]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[16069.193979]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[16069.200129]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16069.207431]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16069.215508]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16069.222199]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16069.227489]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16069.234340]  [<ffffffffffffffff>] 0xffffffffffffffff
[16069.239727] LNet: Service thread pid 109311 was inactive for 1204.39s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[16069.253567] LNet: Skipped 67 previous similar messages
[16121.660800] LustreError: dumping log to /tmp/lustre-log.1562688100.63569
[16150.263297] LNet: Service thread pid 204115 was inactive for 1202.78s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[16150.279402] LNet: Skipped 35 previous similar messages
[16150.285188] LustreError: dumping log to /tmp/lustre-log.1562688129.204115
[16302.060377] Lustre: f2-MDT0003: Client 9fd81d33-2d47-755d-45f0-51b17d65fea2 (at 258@gni5) reconnecting
[16302.071113] Lustre: Skipped 566 previous similar messages
[16302.077307] Lustre: f2-MDT0003: Connection restored to 9fd81d33-2d47-755d-45f0-51b17d65fea2 (at 258@gni5)
[16302.087682] Lustre: Skipped 566 previous similar messages
[16366.203664] LustreError: 424391:0:(service.c:3336:ptlrpc_svcpt_health_check()) mdt: unhealthy - request has been waiting 2089s
[16366.218110] LustreError: 424391:0:(service.c:3336:ptlrpc_svcpt_health_check()) Skipped 17 previous similar messages
[16684.724720] Lustre: 40861:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply
  req@ffff9fdca1220000 x1637715041215312/t0(0) o101->f5576889-4658-e743-7261-945f326093a9@367@gni4:239/0 lens 376/0 e 1 to 0 dl 1562688669 ref 2 fl New:/0/ffffffff rc 0/-1
[16684.759516] Lustre: 40861:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 510 previous similar messages
[16705.969075] LNet: Service thread pid 51800 was inactive for 1202.53s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
[16705.989809] LNet: Skipped 4 previous similar messages
[16705.995656] Pid: 51800, comm: mdt_rdpg04_007 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16706.005872] Call Trace:
[16706.008914]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16706.015700]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16706.021882]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16706.028436]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16706.035061]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16706.042270]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[16706.049232]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[16706.055583]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16706.063096]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16706.071384]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16706.078260]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16706.083746]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16706.090797]  [<ffffffffffffffff>] 0xffffffffffffffff
[16706.096385] LustreError: dumping log to /tmp/lustre-log.1562688686.51800
[16706.106790] Pid: 64065, comm: mdt_rdpg03_018 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16706.118408] Call Trace:
[16706.121776]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16706.128812]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16706.135011]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16706.141573]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16706.148212]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16706.155452]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[16706.162428]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[16706.168782]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16706.176279]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16706.184544]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16706.191417]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16706.196882]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16706.203892]  [<ffffffffffffffff>] 0xffffffffffffffff
[16706.209447] Pid: 204122, comm: mdt_rdpg04_036 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16706.219728] Call Trace:
[16706.222736]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16706.229492]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16706.235630]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16706.242124]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16706.248687]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16706.255837]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[16706.262719]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[16706.268979]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16706.276378]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16706.284533]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16706.291300]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16706.296654]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16706.303551]  [<ffffffffffffffff>] 0xffffffffffffffff
[16706.308974] Pid: 128140, comm: mdt_rdpg02_024 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16706.319134] Call Trace:
[16706.322027]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16706.328650]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16706.334670]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16706.341059]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16706.347525]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16706.354589]  [<ffffffffc1748b02>] top_trans_start+0x702/0x940 [ptlrpc]
[16706.361582]  [<ffffffffc1a92494>] lod_trans_start+0x34/0x40 [lod]
[16706.368129]  [<ffffffffc1b46dda>] mdd_trans_start+0x1a/0x20 [mdd]
[16706.374671]  [<ffffffffc1b3afff>] mdd_attr_set+0x56f/0xca0 [mdd]
[16706.381122]  [<ffffffffc19f7510>] mdt_mfd_close+0x750/0x850 [mdt]
[16706.387650]  [<ffffffffc19fca01>] mdt_close_internal+0x121/0x220 [mdt]
[16706.394610]  [<ffffffffc19fcd20>] mdt_close+0x220/0x780 [mdt]
[16706.400791]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16706.408114]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16706.416215]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16706.422933]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16706.428232]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16706.435088]  [<ffffffffffffffff>] 0xffffffffffffffff
[16706.440478] Pid: 211630, comm: mdt_rdpg03_046 3.10.0-957.5.1.el7.x86_64 #1 SMP Wed Dec 19 10:46:58 EST 2018
[16706.450610] Call Trace:
[16706.453474]  [<ffffffffc09ed325>] cv_wait_common+0x125/0x150 [spl]
[16706.460093]  [<ffffffffc09ed365>] __cv_wait+0x15/0x20 [spl]
[16706.466101]  [<ffffffffc12905fb>] dmu_tx_wait+0x20b/0x3b0 [zfs]
[16706.472464]  [<ffffffffc1290831>] dmu_tx_assign+0x91/0x490 [zfs]
[16706.478914]  [<ffffffffc18ab049>] osd_trans_start+0x199/0x440 [osd_zfs]
[16706.485961]  [<ffffffffc19f4837>] mdt_empty_transno+0xf7/0x850 [mdt]
[16706.492748]  [<ffffffffc19fcd32>] mdt_close+0x232/0x780 [mdt]
[16706.498929]  [<ffffffffc17331da>] tgt_request_handle+0xaea/0x1580 [ptlrpc]
[16706.506242]  [<ffffffffc16d880b>] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc]
[16706.514344]  [<ffffffffc16dc13c>] ptlrpc_main+0xafc/0x1fc0 [ptlrpc]
[16706.521062]  [<ffffffff9b0c1c71>] kthread+0xd1/0xe0
[16706.526362]  [<ffffffff9b774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[16706.533217]  [<ffffffffffffffff>] 0xffffffffffffffff
[16706.538608] LNet: Service thread pid 39037 was inactive for 1203.17s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
[16706.552365] LNet: Skipped 11 previous similar messages
[16759.087969] LustreError: dumping log to /tmp/lustre-log.1562688739.421986
[16787.690437] LustreError: dumping log to /tmp/lustre-log.1562688768.63240
[16900.654199] Lustre: f2-MDT0003: Client c88b67d7-6af0-eab5-5731-86d50b4e1e02 (at 747@gni5) reconnecting
[16900.664940] Lustre: Skipped 612 previous similar messages
[16900.671830] Lustre: f2-MDT0003: Connection restored to c88b67d7-6af0-eab5-5731-86d50b4e1e02 (at 747@gni5)
[16900.682024] Lustre: Skipped 612 previous similar messages
[16921.115631] SysRq : Trigger a crash
[16921.120652] BUG: unable to handle kernel NULL pointer dereference at           (null)
[16921.129646] IP: [<ffffffff9b462276>] sysrq_handle_crash+0x16/0x20
[16921.136457] PGD 8000005d7ba70067 PUD 5d4c2cf067 PMD 0 
[16921.142104] Oops: 0002 [#1] SMP 
[16921.145813] Modules linked in: osp(OE) mdd(OE) lod(OE) mdt(OE) lfsck(OE) mgc(OE) osd_zfs(OE) lquota(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) zfs(POE) zunicode(POE) zlua(POE) zcommon(POE) znvpair(POE) zavl(POE) icp(POE) spl(OE) dm_round_robin dm_multipath ko2iblnd(OE) lnet(OE) libcfs(OE) xt_comment xt_multiport nfsv3 nfs_acl mpt2sas mptctl mptbase ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter dell_rbu ib_isert iscsi_target_mod target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm rpcrdma rdma_ucm ib_uverbs ib_umad ib_iser ib_ipoib rdma_cm iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_core dm_mirror dm_region_hash dm_log dm_mod sd_mod crc_t10dif
[16921.221392]  crct10dif_generic sg iTCO_wdt dell_smbios iTCO_vendor_support dell_wmi_descriptor dcdbas skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crct10dif_pclmul crct10dif_common crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mpt3sas raid_class scsi_transport_sas mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm drm_panel_orientation_quirks ahci mei_me libahci i2c_i801 lpc_ich mei libata wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_pad acpi_power_meter binfmt_misc ip_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache crc32c_intel mlx5_core igb mlxfw i2c_algo_bit devlink dca ptp pps_core sunrpc [last unloaded: zunicode]
[16921.293925] CPU: 0 PID: 0 Comm: swapper/0 Kdump: loaded Tainted: P           OE  ------------ T 3.10.0-957.5.1.el7.x86_64 #1
[16921.305734] Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 1.6.13 12/17/2018
[16921.313912] task: ffffffff9bc18480 ti: ffffffff9bc00000 task.ti: ffffffff9bc00000
[16921.322005] RIP: 0010:[<ffffffff9b462276>]  [<ffffffff9b462276>] sysrq_handle_crash+0x16/0x20
[16921.331171] RSP: 0018:ffff9fe139a03d98  EFLAGS: 00010046
[16921.337109] RAX: ffffffff9b462260 RBX: ffffffff9bce4c60 RCX: 0000000000000000
[16921.344862] RDX: 0000000000000000 RSI: ffff9fe139a13898 RDI: 0000000000000063
[16921.352602] RBP: ffff9fe139a03d98 R08: ffffffff9bfe38bc R09: ffff9fe10de56f00
[16921.360347] R10: 00000000000015a0 R11: 000000000000159f R12: 0000000000000063
[16921.368094] R13: 0000000000000001 R14: 0000000000000007 R15: 0000000000000063
[16921.375841] FS:  0000000000000000(0000) GS:ffff9fe139a00000(0000) knlGS:0000000000000000
[16921.384540] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[16921.390912] CR2: 0000000000000000 CR3: 0000005d76a7a000 CR4: 00000000007607f0
[16921.398665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[16921.406421] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[16921.414168] PKRU: 00000000
[16921.417496] Call Trace:
[16921.420565]  <IRQ> 
[16921.422496]  [<ffffffff9b462a9d>] __handle_sysrq+0x10d/0x170
[16921.429575]  [<ffffffff9b462b26>] handle_sysrq+0x26/0x30
[16921.435499]  [<ffffffff9b47b837>] serial8250_rx_chars+0xf7/0x200
[16921.442104]  [<ffffffff9b47b9aa>] serial8250_handle_irq.part.13+0x6a/0xb0
[16921.449471]  [<ffffffff9b47ba3d>] serial8250_default_handle_irq+0x2d/0x30
[16921.456834]  [<ffffffff9b47a988>] serial8250_interrupt+0x68/0xe0
[16921.463416]  [<ffffffff9b14a544>] __handle_irq_event_percpu+0x44/0x1c0
[16921.470524]  [<ffffffff9b14a6f2>] handle_irq_event_percpu+0x32/0x80
[16921.477362]  [<ffffffff9b14a77c>] handle_irq_event+0x3c/0x60
[16921.483587]  [<ffffffff9b14d52f>] handle_edge_irq+0x7f/0x150
[16921.489804]  [<ffffffff9b02e554>] handle_irq+0xe4/0x1a0
[16921.495578]  [<ffffffff9b0da05e>] ? sched_clock_idle_wakeup_event+0x1e/0x20
[16921.503083]  [<ffffffff9b10bc0d>] ? tick_check_idle+0xbd/0xd0
[16921.509369]  [<ffffffff9b7795dd>] do_IRQ+0x4d/0xf0
[16921.514703]  [<ffffffff9b76b362>] common_interrupt+0x162/0x162
[16921.521072]  <EOI> 
[16921.523006]  [<ffffffff9b5ae657>] ? cpuidle_enter_state+0x57/0xd0
[16921.530360]  [<ffffffff9b5ae64d>] ? cpuidle_enter_state+0x4d/0xd0
[16921.536965]  [<ffffffff9b5ae7ae>] cpuidle_idle_call+0xde/0x230
[16921.543293]  [<ffffffff9b0366de>] arch_cpu_idle+0xe/0xc0
[16921.549096]  [<ffffffff9b0fc4da>] cpu_startup_entry+0x14a/0x1e0
[16921.555485]  [<ffffffff9b74ff37>] rest_init+0x77/0x80
[16921.561001]  [<ffffffff9bd861c6>] start_kernel+0x44b/0x46c
[16921.566931]  [<ffffffff9bd85b7b>] ? repair_env_string+0x5c/0x5c
[16921.573286]  [<ffffffff9bd85120>] ? early_idt_handler_array+0x120/0x120
[16921.580316]  [<ffffffff9bd8572f>] x86_64_start_reservations+0x24/0x26
[16921.587163]  [<ffffffff9bd85885>] x86_64_start_kernel+0x154/0x177
[16921.593646]  [<ffffffff9b0000d5>] start_cpu+0x5/0x14
[16921.598996] Code: eb 9b 45 01 f4 45 39 65 34 75 e5 4c 89 ef e8 e2 f7 ff ff eb db 0f 1f 44 00 00 55 48 89 e5 c7 05 a1 50 7e 00 01 00 00 00 0f ae f8 <c6> 04 25 00 00 00 00 01 5d c3 0f 1f 44 00 00 55 31 c0 c7 05 1e 
[16921.620213] RIP  [<ffffffff9b462276>] sysrq_handle_crash+0x16/0x20
[16921.626794]  RSP <ffff9fe139a03d98>
[16921.630659] CR2: 0000000000000000
