Installed Packages [root@zlfs2-mds1 ~]# rpm -qa | grep lustre | sort -u kernel-3.10.0-327.22.2.el7_lustre.x86_64 kernel-debuginfo-3.10.0-327.22.2.el7_lustre.x86_64 kernel-debuginfo-common-x86_64-3.10.0-327.22.2.el7_lustre.x86_64 kernel-devel-3.10.0-327.22.2.el7_lustre.x86_64 kernel-headers-3.10.0-327.22.2.el7_lustre.x86_64 kernel-tools-3.10.0-327.22.2.el7_lustre.x86_64 kernel-tools-debuginfo-3.10.0-327.22.2.el7_lustre.x86_64 kernel-tools-libs-3.10.0-327.22.2.el7_lustre.x86_64 kernel-tools-libs-devel-3.10.0-327.22.2.el7_lustre.x86_64 kmod-lustre-2.8.56_26_g6fad3ab-1.el7.x86_64 kmod-lustre-osd-ldiskfs-2.8.56_26_g6fad3ab-1.el7.x86_64 kmod-lustre-osd-zfs-2.8.56_26_g6fad3ab-1.el7.x86_64 kmod-lustre-tests-2.8.56_26_g6fad3ab-1.el7.x86_64 kmod-spl-3.10.0-327.22.2.el7_lustre.x86_64-0.6.5.7-1.el7.x86_64 kmod-spl-devel-3.10.0-327.22.2.el7_lustre.x86_64-0.6.5.7-1.el7.x86_64 kmod-zfs-3.10.0-327.22.2.el7_lustre.x86_64-0.6.5.7-1.el7.x86_64 kmod-zfs-devel-3.10.0-327.22.2.el7_lustre.x86_64-0.6.5.7-1.el7.x86_64 lustre-2.8.56_26_g6fad3ab-1.el7.x86_64 lustre-debuginfo-2.8.56_26_g6fad3ab-1.el7.x86_64 lustre-iokit-2.8.56_26_g6fad3ab-1.el7.x86_64 lustre-osd-ldiskfs-mount-2.8.56_26_g6fad3ab-1.el7.x86_64 lustre-osd-zfs-mount-2.8.56_26_g6fad3ab-1.el7.x86_64 lustre-tests-2.8.56_26_g6fad3ab-1.el7.x86_64 perf-3.10.0-327.22.2.el7_lustre.x86_64 perf-debuginfo-3.10.0-327.22.2.el7_lustre.x86_64 python-perf-3.10.0-327.22.2.el7_lustre.x86_64 python-perf-debuginfo-3.10.0-327.22.2.el7_lustre.x86_64 [root@lfsmaster ~]# ssh z2m1 Last login: Tue Aug 16 21:50:30 2016 from 10.10.100.99 [root@zlfs2-mds1 ~]# lctl list_nids 192.168.5.21@o2ib [root@zlfs2-mds1 ~]# lctl ping 192.168.5.31@o2ib 12345-0@lo 12345-192.168.5.31@o2ib [root@zlfs2-mds1 ~]# exit logout Connection to z2m1 closed. [root@lfsmaster ~]# ssh z2o1 Last login: Tue Aug 16 21:50:51 2016 from 10.10.100.99 [root@zlfs2-oss1 ~]# lctl list_nids 192.168.5.31@o2ib [root@zlfs2-oss1 ~]# lctl ping 192.168.5.21@o2ib 12345-0@lo 12345-192.168.5.21@o2ib [root@zlfs2-oss1 ~]# mount -vvv -t lustre /dev/nvme0n1 /mnt/MDT0000 arg[0] = /sbin/mount.lustre arg[1] = -v arg[2] = -o arg[3] = rw arg[4] = /dev/nvme0n1 arg[5] = /mnt/MDT0000 source = /dev/nvme0n1 (/dev/nvme0n1), target = /mnt/MDT0000 options = rw checking for existing Lustre data: found Reading CONFIGS/mountdata Writing CONFIGS/mountdata mounting device /dev/nvme0n1 at /mnt/MDT0000, flags=0x1000000 options=osd=osd-ldiskfs,user_xattr,errors=remount-ro,mgsnode=192.168.5.21@o2ib,virgin,update,param=mgsnode=192.168.5.21@o2ib,svname=zlfs2-MDT0000,device=/dev/nvme0n1 mount.lustre: cannot parse scheduler options for '/sys/block/nvme0n1/queue/scheduler' Message from syslogd@zlfs2-oss1 at Aug 16 21:52:33 ... kernel:LustreError: 3842:0:(lu_object.c:1243:lu_device_fini()) ASSERTION( atomic_read(&d->ld_ref) == 0 ) failed: Refcount is 1 Message from syslogd@zlfs2-oss1 at Aug 16 21:52:33 ... kernel:LustreError: 3842:0:(lu_object.c:1243:lu_device_fini()) LBUG Message from syslogd@zlfs2-oss1 at Aug 16 21:52:33 ... kernel:Kernel panic - not syncing: LBUG ######### /var/log/messages output at time of kernel panic ############ [root@zlfs2-oss1 ~]# tail -f /var/log/messages Aug 16 21:51:43 zlfs2-oss1 kernel: Lustre: Failing over zlfs2-MDT0000 Aug 16 21:51:49 zlfs2-oss1 kernel: Lustre: zlfs2-MDT0000: Not available for connect from 192.168.5.21@o2ib (stopping) Aug 16 21:51:49 zlfs2-oss1 kernel: LustreError: 3732:0:(import.c:338:ptlrpc_invalidate_import()) zlfs2-OST0004_UUID: rc = -110 waiting for callback (1 != 0) Aug 16 21:51:49 zlfs2-oss1 kernel: LustreError: 3732:0:(import.c:364:ptlrpc_invalidate_import()) @@@ still on sending list req@ffff881fe3818300 x1542853838766384/t0(0) o8->zlfs2-OST0004-osc-MDT0000@192.168.5.22@o2ib:28/4 lens 520/544 e 0 to 0 dl 1471380708 ref 2 fl UnregRPC:EN/0/ffffffff rc -5/-1 Aug 16 21:51:49 zlfs2-oss1 kernel: LustreError: 3732:0:(import.c:378:ptlrpc_invalidate_import()) zlfs2-OST0004_UUID: Unregistering RPCs found (1). Network is sluggish? Waiting them to error out. Aug 16 21:52:14 zlfs2-oss1 kernel: Lustre: zlfs2-MDT0000: Not available for connect from 192.168.5.21@o2ib (stopping) Aug 16 21:52:14 zlfs2-oss1 kernel: Lustre: Skipped 3 previous similar messages Aug 16 21:52:16 zlfs2-oss1 systemd-logind: New session 6 of user root. Aug 16 21:52:16 zlfs2-oss1 systemd: Started Session 6 of user root. Aug 16 21:52:16 zlfs2-oss1 systemd: Starting Session 6 of user root. Aug 16 21:52:33 zlfs2-oss1 kernel: LustreError: 3842:0:(lu_object.c:1243:lu_device_fini()) ASSERTION( atomic_read(&d->ld_ref) == 0 ) failed: Refcount is 1 Message from syslogd@zlfs2-oss1 at Aug 16 21:52:33 ... kernel:LustreError: 3842:0:(lu_object.c:1243:lu_device_fini()) ASSERTION( atomic_read(&d->ld_ref) == 0 ) failed: Refcount is 1 Aug 16 21:52:33 zlfs2-oss1 kernel: LustreError: 3842:0:(lu_object.c:1243:lu_device_fini()) LBUG Aug 16 21:52:33 zlfs2-oss1 kernel: Pid: 3842, comm: osp-syn-4-0 Aug 16 21:52:33 zlfs2-oss1 kernel: #012Call Trace: Aug 16 21:52:33 zlfs2-oss1 kernel: [] libcfs_debug_dumpstack+0x53/0x80 [libcfs] Aug 16 21:52:33 zlfs2-oss1 kernel: [] lbug_with_loc+0x45/0xc0 [libcfs] Aug 16 21:52:33 zlfs2-oss1 kernel: [] lu_device_fini+0xb8/0xc0 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] ls_device_put+0x82/0x2a0 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] local_oid_storage_fini+0xdd/0x210 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] llog_osd_cleanup+0x3c/0x50 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] __llog_ctxt_put+0x93/0x140 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] llog_cleanup+0xc3/0x490 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] ? llog_handle_put+0x2d/0x70 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] ? llog_close+0x59/0x1a0 [obdclass] Aug 16 21:52:33 zlfs2-oss1 kernel: [] osp_sync_thread+0x46b/0x990 [osp] Message from syslogd@zlfs2-oss1 at Aug 16 21:52:33 ... kernel:LustreError: 3842:0:(lu_object.c:1243:lu_device_fini()) LBUG Aug 16 21:52:33 zlfs2-oss1 kernel: [] ? osp_sync_thread+0x0/0x990 [osp] Aug 16 21:52:33 zlfs2-oss1 kernel: [] kthread+0xcf/0xe0 Aug 16 21:52:33 zlfs2-oss1 kernel: [] ? kthread+0x0/0xe0 Aug 16 21:52:33 zlfs2-oss1 kernel: [] ret_from_fork+0x58/0x90 Aug 16 21:52:33 zlfs2-oss1 kernel: [] ? kthread+0x0/0xe0 Aug 16 21:52:33 zlfs2-oss1 kernel: Aug 16 21:52:33 zlfs2-oss1 kernel: Kernel panic - not syncing: LBUG Message from syslogd@zlfs2-oss1 at Aug 16 21:52:33 ... kernel:Kernel panic - not syncing: LBUG Aug 16 21:52:33 zlfs2-oss1 kernel: CPU: 16 PID: 3842 Comm: osp-syn-4-0 Tainted: P OE ------------ 3.10.0-327.22.2.el7_lustre.x86_64 #1 Aug 16 21:52:33 zlfs2-oss1 kernel: Hardware name: Intel Corporation S2600WTT/S2600WTT, BIOS SE5C610.86B.01.01.0016.033120161139 03/31/2016 Aug 16 21:52:33 zlfs2-oss1 kernel: ffffffffa0bbfdef 000000008386d797 ffff880ff7ba7bf8 ffffffff81636324 Aug 16 21:52:33 zlfs2-oss1 kernel: ffff880ff7ba7c78 ffffffff8162fb9a ffffffff00000008 ffff880ff7ba7c88 Aug 16 21:52:33 zlfs2-oss1 kernel: ffff880ff7ba7c28 000000008386d797 ffffffffa0d0d1d5 0000000000000000 Aug 16 21:52:33 zlfs2-oss1 kernel: Call Trace: Aug 16 21:52:33 zlfs2-oss1 kernel: [] dump_stack+0x19/0x1b Aug 16 21:52:33 zlfs2-oss1 kernel: [] panic+0xd8/0x1e7 Aug 16 21:52:33 zlfs2-oss1 kernel: [] lbug_with_loc+0xab/0xc0 [libcfs] Aug 16 21:52:33 zlfs2-oss1 kernel: [] lu_device_fini+0xb8/0xc0 [obdclass] ########### Apendix ############### [root@lfsmaster FORMAT_SCRIPTS]# cat zlfs2-dne2-ldiskfs_1MDT.sh #!/bin/bash MGSNID=192.168.5.21@o2ib0 FSNAME=zlfs2 RECORDSIZE=1024k ssh zlfs2-mds1 zpool destroy MGT0000 ssh zlfs2-mds1 zpool create MGT0000 mirror -f /dev/sdc /dev/sdd ssh zlfs2-mds1 mkfs.lustre --reformat --mgs --backfstype=zfs MGT0000/MGT0000 ssh zlfs2-mds1 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=0 /dev/nvme0n1 ssh zlfs2-mds1 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=1 /dev/nvme1n1 ssh zlfs2-mds1 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=2 /dev/nvme2n1 ssh zlfs2-mds1 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=3 /dev/nvme3n1 ssh zlfs2-mds2 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=4 /dev/nvme0n1 ssh zlfs2-mds2 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=5 /dev/nvme1n1 ssh zlfs2-mds2 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=6 /dev/nvme2n1 ssh zlfs2-mds2 mkfs.lustre --reformat --ost --fsname=${FSNAME} --mgsnode=${MGSNID} --index=7 /dev/nvme3n1 # ssh zlfs2-oss1 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=0 /dev/nvme0n1 ssh zlfs2-oss2 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=1 /dev/nvme0n1 ssh zlfs2-oss3 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=2 /dev/nvme0n1 ssh zlfs2-oss4 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=3 /dev/nvme0n1 ssh zlfs2-oss5 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=4 /dev/nvme0n1 ssh zlfs2-oss6 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=5 /dev/nvme0n1 ssh zlfs2-oss7 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=6 /dev/nvme0n1 ssh zlfs2-oss8 mkfs.lustre --reformat --mdt --fsname=${FSNAME} --mgsnode=${MGSNID} --index=7 /dev/nvme0n1 #################### [root@lfsmaster FORMAT_SCRIPTS]# ./zlfs2-dne2-ldiskfs_1MDT.sh Permanent disk data: Target: MGS Index: unassigned Lustre FS: Mount type: zfs Flags: 0x64 (MGS first_time update ) Persistent mount opts: Parameters: mkfs_cmd = zfs create -o canmount=off -o xattr=sa MGT0000/MGT0000 Writing MGT0000/MGT0000 properties lustre:version=1 lustre:flags=100 lustre:index=65535 lustre:svname=MGS Permanent disk data: Target: zlfs2:OST0000 Index: 0 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:OST0000 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0000 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme0n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0001 Index: 1 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme1n1 target name zlfs2:OST0001 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0001 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme1n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0002 Index: 2 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme2n1 target name zlfs2:OST0002 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0002 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme2n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0003 Index: 3 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme3n1 target name zlfs2:OST0003 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0003 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme3n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0004 Index: 4 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:OST0004 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0004 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme0n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0005 Index: 5 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme1n1 target name zlfs2:OST0005 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0005 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme1n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0006 Index: 6 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme2n1 target name zlfs2:OST0006 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0006 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme2n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:OST0007 Index: 7 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x62 (OST first_time update ) Persistent mount opts: ,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 763097MB formatting backing filesystem ldiskfs on /dev/nvme3n1 target name zlfs2:OST0007 4k blocks 195353046 options -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:OST0007 -J size=400 -I 256 -i 69905 -q -O extents,uninit_bg,dir_nlink,quota,huge_file,flex_bg -G 256 -E resize="4290772992",lazy_journal_init -F /dev/nvme3n1 195353046 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0000 Index: 0 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0000 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0000 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0001 Index: 1 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0001 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0001 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0002 Index: 2 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0002 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0002 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0003 Index: 3 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0003 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0003 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0004 Index: 4 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0004 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0004 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0005 Index: 5 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0005 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0005 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0006 Index: 6 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0006 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0006 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata Permanent disk data: Target: zlfs2:MDT0007 Index: 7 Lustre FS: zlfs2 Mount type: ldiskfs Flags: 0x61 (MDT first_time update ) Persistent mount opts: user_xattr,errors=remount-ro Parameters: mgsnode=192.168.5.21@o2ib device size = 1907729MB formatting backing filesystem ldiskfs on /dev/nvme0n1 target name zlfs2:MDT0007 4k blocks 488378646 options -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F mkfs_cmd = mke2fs -j -b 4096 -L zlfs2:MDT0007 -J size=4096 -I 512 -i 2048 -q -O dirdata,uninit_bg,^extents,dir_nlink,quota,huge_file,flex_bg -E lazy_journal_init -F /dev/nvme0n1 488378646 Writing CONFIGS/mountdata