<!-- 
RSS generated by JIRA (9.4.14#940014-sha1:734e6822bbf0d45eff9af51f82432957f73aa32c) at Sat Feb 10 02:18:05 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>Whamcloud Community JIRA</title>
    <link>https://jira.whamcloud.com</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.4.14</version>
        <build-number>940014</build-number>
        <build-date>05-12-2023</build-date>
    </build-info>


<item>
            <title>[LU-8498] configuration from log &apos;nodemap&apos; failed (-22)</title>
                <link>https://jira.whamcloud.com/browse/LU-8498</link>
                <project id="10000" key="LU">Lustre</project>
                    <description>&lt;p&gt; In current master 2.8.56, a newly created file system failed at mounting OST due to nodemap log error.&lt;/p&gt;

&lt;p&gt;From the log message:&lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;00000100:00000001:3.0:1470938259.243549:0:9524:0:(client.c:1052:ptlrpc_set_destroy()) Process leaving
00000100:00000001:3.0:1470938259.243549:0:9524:0:(client.c:2896:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0)
10000000:00000001:3.0:1470938259.243551:0:9524:0:(mgc_request.c:1716:mgc_process_recover_nodemap_log()) Process leaving via out (rc=18446744073709551594 : -22 : 0xffffffffffffffea)
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;it looks like the corresponding log has a zero size that triggered this error.&lt;/p&gt;

&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;        &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt; (ealen == 0) { &lt;span class=&quot;code-comment&quot;&gt;/* no logs transferred */&lt;/span&gt;
#ifdef HAVE_SERVER_SUPPORT
                &lt;span class=&quot;code-comment&quot;&gt;/* config changed since first read RPC */&lt;/span&gt;
                &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt; (cld_is_nodemap(cld) &amp;amp;&amp;amp; config_read_offset == 0) {
                        recent_nodemap = NULL;
                        nodemap_config_dealloc(new_config);
                        new_config = NULL;

                        CDEBUG(D_INFO, &lt;span class=&quot;code-quote&quot;&gt;&quot;nodemap config changed in transit, retrying\n&quot;&lt;/span&gt;);

                        &lt;span class=&quot;code-comment&quot;&gt;/* setting eof to &lt;span class=&quot;code-keyword&quot;&gt;false&lt;/span&gt;, we request config again */&lt;/span&gt;
                        eof = &lt;span class=&quot;code-keyword&quot;&gt;false&lt;/span&gt;;
                        GOTO(out, rc = 0);
                }
#endif
                &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt; (!eof)
                        rc = -EINVAL;
                GOTO(out, rc);
        }
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;We have a debug log and will attach it soon.&lt;/p&gt;</description>
                <environment></environment>
        <key id="38794">LU-8498</key>
            <summary>configuration from log &apos;nodemap&apos; failed (-22)</summary>
                <type id="1" iconUrl="https://jira.whamcloud.com/secure/viewavatar?size=xsmall&amp;avatarId=11303&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.whamcloud.com/images/icons/priorities/critical.svg">Critical</priority>
                        <status id="5" iconUrl="https://jira.whamcloud.com/images/icons/statuses/resolved.png" description="A resolution has been taken, and it is awaiting verification by reporter. From here issues are either reopened, or are closed.">Resolved</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="1">Fixed</resolution>
                                        <assignee username="kit.westneat">Kit Westneat</assignee>
                                    <reporter username="jay">Jinshan Xiong</reporter>
                        <labels>
                            <label>LS_RZ</label>
                    </labels>
                <created>Thu, 11 Aug 2016 18:11:45 +0000</created>
                <updated>Thu, 14 Jun 2018 21:39:07 +0000</updated>
                            <resolved>Thu, 8 Sep 2016 04:19:20 +0000</resolved>
                                                    <fixVersion>Lustre 2.9.0</fixVersion>
                                        <due></due>
                            <votes>0</votes>
                                    <watches>6</watches>
                                                                            <comments>
                            <comment id="161663" author="jsalians_intel" created="Thu, 11 Aug 2016 18:29:48 +0000"  >&lt;p&gt;Currently this is blocking testing of: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8460&quot; title=&quot;(osc_cache.c:658:osc_extent_find()) ASSERTION( (max_pages &amp;amp; ~chunk_mask) == 0 ) failed: LBUG&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8460&quot;&gt;&lt;del&gt;LU-8460&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;http://review.whamcloud.com/#/c/21891/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/#/c/21891/&lt;/a&gt;  &lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;lctl set_param debug=-1&lt;br/&gt;
debug=-1&lt;/li&gt;
	&lt;li&gt;lctl dk clear&lt;br/&gt;
Debug log: 4741 lines, 4741 kept, 0 dropped, 0 bad.&lt;/li&gt;
	&lt;li&gt;mkfs.lustre --backfstype=zfs  --reformat --replace --fsname=lsdraid --ost --index=0 --mgsnode=192.168.1.5@o2ib ost0/ost0&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;   Permanent disk data:&lt;br/&gt;
Target:     lsdraid-OST0000&lt;br/&gt;
Index:      0&lt;br/&gt;
Lustre FS:  lsdraid&lt;br/&gt;
Mount type: zfs&lt;br/&gt;
Flags:      0x42&lt;br/&gt;
              (OST update )&lt;br/&gt;
Persistent mount opts: &lt;br/&gt;
Parameters: mgsnode=192.168.1.5@o2ib&lt;/p&gt;

&lt;p&gt;mkfs_cmd = zfs create -o canmount=off -o xattr=sa ost0/ost0&lt;br/&gt;
Writing ost0/ost0 properties&lt;br/&gt;
  lustre:version=1&lt;br/&gt;
  lustre:flags=66&lt;br/&gt;
  lustre:index=0&lt;br/&gt;
  lustre:fsname=lsdraid&lt;br/&gt;
  lustre:svname=lsdraid-OST0000&lt;br/&gt;
  lustre:mgsnode=192.168.1.5@o2ib&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;lctl dk clear&lt;br/&gt;
Debug log: 7281 lines, 7281 kept, 0 dropped, 0 bad.&lt;/li&gt;
&lt;/ol&gt;


&lt;ol&gt;
	&lt;li&gt;mount -t lustre ost0/ost0 /mnt/lustre/ost0&lt;br/&gt;
mount.lustre: mount ost0/ost0 at /mnt/lustre/ost0 failed: No such file or directory&lt;br/&gt;
Is the MGS specification correct?&lt;br/&gt;
Is the filesystem name correct?&lt;br/&gt;
If upgrading, is the copied client log valid? (see upgrade docs)&lt;/li&gt;
&lt;/ol&gt;
</comment>
                            <comment id="161747" author="kit.westneat" created="Fri, 12 Aug 2016 14:58:40 +0000"  >&lt;p&gt;It looks like the MGS hasn&apos;t created the nodemap config when the OST is mounting. Can you get the MGS logs as well? Or is there a maloo link for this failure? I&apos;m not able to see patch 21891 for some reason.&lt;/p&gt;</comment>
                            <comment id="161772" author="pjones" created="Fri, 12 Aug 2016 17:58:03 +0000"  >&lt;p&gt;Kit&lt;/p&gt;

&lt;p&gt;Don&apos;t worry - 21891 is just a port to of the &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8460&quot; title=&quot;(osc_cache.c:658:osc_extent_find()) ASSERTION( (max_pages &amp;amp; ~chunk_mask) == 0 ) failed: LBUG&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8460&quot;&gt;&lt;del&gt;LU-8460&lt;/del&gt;&lt;/a&gt; fix to another branch so not important&lt;/p&gt;

&lt;p&gt;Peter&lt;/p&gt;</comment>
                            <comment id="161796" author="jsalians_intel" created="Fri, 12 Aug 2016 21:51:57 +0000"  >&lt;p&gt;This log covers from: mount -t lustre zfspool/mdt1 /mnt/mdt to when the OST fails to connect.&lt;/p&gt;</comment>
                            <comment id="161807" author="jsalians_intel" created="Sat, 13 Aug 2016 00:24:07 +0000"  >&lt;p&gt;This seems like a pretty serious bug since this feature landed I have been unable to use master at all as I cannot get any OST to connect to MDS.  Is there any work around?  CORAL testing has completely stopped because of this. &lt;/p&gt;</comment>
                            <comment id="161866" author="kit.westneat" created="Mon, 15 Aug 2016 04:37:10 +0000"  >&lt;p&gt;Hi John,&lt;/p&gt;

&lt;p&gt;Thanks for the logs. I took a quick look, but there&apos;s nothing obvious. The MDS says it&apos;s sending over a 1MB config RPC, so I&apos;m not sure why the MGC thinks it&apos;s not getting anything. I&apos;ll take a closer look tomorrow.&lt;/p&gt;

&lt;p&gt;Can you confirm you are just running straight master, no patches? FWIW line 1716 doesn&apos;t correspond to a GOTO statement on the tip of master for me (hash 6fad3ab). &lt;/p&gt;

&lt;p&gt;You could try changing the return code from -EINVAL to 0 on that eof check as a workaround. It shouldn&apos;t cause any problems to receive a 0 length RPC if you aren&apos;t using nodemap, but it also shouldn&apos;t happen as far as I understand it. Here&apos;s the eof check I mean:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;                &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt; (!eof)
                        rc = 0;
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;What does your test setup look like? Is there any way to reproduce the failure in maloo? &lt;/p&gt;

&lt;p&gt;When you say &quot;since this feature landed I have been unable to use master&quot; can you clarify which feature you mean? There have been a number of patches related to nodemap config transfer that have landed in the past couple of months. If you could specify the last version (commit hash) that worked, and the first version that didn&apos;t, that would be helpful.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Kit&lt;/p&gt;</comment>
                            <comment id="161896" author="jsalians_intel" created="Mon, 15 Aug 2016 15:40:48 +0000"  >&lt;p&gt;Good question &amp;#8211; I reproduced this issue this morning on the following: &lt;br/&gt;
-b 0 -j zfs-c-p-lustre-patch-vanillais review which my build 21907.3&quot; or build 116 &amp;#8211; however I believe this  builds what ever was submitted for review last.&lt;/p&gt;

&lt;p&gt;mount.lustre: mount ost0/ost0 at /mnt/lustre/ost0 failed: No such file or directory&lt;br/&gt;
Is the MGS specification correct?&lt;br/&gt;
Is the filesystem name correct?&lt;/p&gt;

&lt;p&gt;I am installing the tip of master now to see if it has the same issue. &lt;/p&gt;</comment>
                            <comment id="161919" author="jsalians_intel" created="Mon, 15 Aug 2016 18:53:08 +0000"  >&lt;p&gt;I tried this on master: &lt;/p&gt;

&lt;p&gt;(no draid &amp;#8211; this is not using our prototype branch) &lt;br/&gt;
zpool create -f -o cachefile=none -O recordsize=16MB ost0 raidz1  /dev/mapper/mpathaj /dev/mapper/mpathai /dev/mapper/mpathah /dev/mapper/mpathag /dev/mapper/mpathaq /dev/mapper/mpathap /dev/mapper/mpathak /dev/mapper/mpathz /dev/mapper/mpatham /dev/mapper/mpathal /dev/mapper/mpathao&lt;br/&gt;
mkfs.lustre --backfstype=zfs  --reformat --replace --fsname=lsdraid --ost --index=0 --mgsnode=192.168.1.5@o2ib ost0/ost0&lt;br/&gt;
mount -t lustre ost0/ost0 /mnt/lustre/ost0&lt;br/&gt;
mount.lustre: mount ost0/ost0 at /mnt/lustre/ost0 failed: No such file or directory&lt;br/&gt;
Is the MGS specification correct?&lt;br/&gt;
Is the filesystem name correct?&lt;br/&gt;
If upgrading, is the copied client log valid? (see upgrade docs)&lt;/p&gt;

&lt;p&gt;I repeated the mount with lcdl debug enabled and attached here.  This build was with  lustre master: &lt;/p&gt;

&lt;p&gt;Git Build Data&lt;/p&gt;

&lt;p&gt;Revision: 6fad3abf6f962d04989422cb44dfb7aa0835ad07&lt;br/&gt;
refs/remotes/origin/master&lt;br/&gt;
Built Branches&lt;/p&gt;

&lt;p&gt;refs/remotes/origin/master: Build #170 of Revision 6fad3abf6f962d04989422cb44dfb7aa0835ad07 (refs/remotes/origin/master)&lt;/p&gt;


&lt;p&gt;Fetching changes from the remote Git repository&lt;br/&gt;
 &amp;gt; git config remote.origin.url ssh://hudson@review.whamcloud.com:29418/fs/lustre-release # timeout=10&lt;br/&gt;
Fetching upstream changes from ssh://hudson@review.whamcloud.com:29418/fs/lustre-release&lt;br/&gt;
 &amp;gt; git --version # timeout=10&lt;br/&gt;
 &amp;gt; git -c core.askpass=true fetch --tags --progress ssh://hudson@review.whamcloud.com:29418/fs/lustre-release +refs/heads/&lt;b&gt;:refs/remotes/origin/&lt;/b&gt; --depth=1&lt;br/&gt;
Checking out Revision 6fad3abf6f962d04989422cb44dfb7aa0835ad07 (refs/remotes/origin/master)&lt;br/&gt;
 &amp;gt; git config core.sparsecheckout # timeout=10&lt;br/&gt;
 &amp;gt; git checkout -f 6fad3abf6f962d04989422cb44dfb7aa0835ad07&lt;br/&gt;
 &amp;gt; git rev-list 3ed9f9a0b43bc48cf778539f1281cd60332b99d3 # timeout=10&lt;br/&gt;
 &amp;gt; git tag -a -f -m Jenkins Build #170 jenkins-arch=x86_64,build_type=client,distro=el7.2,ib_stack=inkernel-170 # timeout=10&lt;br/&gt;
Checking out Revision 6fad3abf6f962d04989422cb44dfb7aa0835ad07 (refs/remotes/origin/master)&lt;br/&gt;
 &amp;gt; git config core.sparsecheckout # timeout=10&lt;br/&gt;
 &amp;gt; git checkout -f 6fad3abf6f962d04989422cb44dfb7aa0835ad07&lt;br/&gt;
 &amp;gt; git rev-list 3ed9f9a0b43bc48cf778539f1281cd60332b99d3 # timeout=10&lt;br/&gt;
 &amp;gt; git tag -a -f -m Jenkins Build #170 jenkins-zfs-lustre-master-vanilla-170 # timeout=10 &lt;/p&gt;

&lt;p&gt;Our build engineer noted: build 170 was triggered by &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-7899&quot; title=&quot;osd_xattr_set() to batch actual EA update&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-7899&quot;&gt;&lt;del&gt;LU-7899&lt;/del&gt;&lt;/a&gt;, which was the last SCM change. &lt;/p&gt;</comment>
                            <comment id="161924" author="jsalians_intel" created="Mon, 15 Aug 2016 20:03:58 +0000"  >&lt;p&gt;I should say the lustre messages look the same to me still: [  861.004988] LustreError: 33234:0:(mgc_request.c:257:do_config_log_add()) MGC192.168.1.5@o2ib: failed processing log, type 4: rc = -22&lt;br/&gt;
[  861.020257] LustreError: 13a-8: Failed to get MGS log lsdraid-OST0000 and no local copy.&lt;br/&gt;
[  861.030603] LustreError: 15c-8: MGC192.168.1.5@o2ib: The configuration from log &apos;lsdraid-OST0000&apos; failed (-2). This may be the result of communication errors between this node and the MGS, a bad configuration, or other errors. See the syslog for more information.&lt;br/&gt;
[  861.060862] LustreError: 33234:0:(obd_mount_server.c:1352:server_start_targets()) failed to start server lsdraid-OST0000: -2&lt;br/&gt;
[  861.075061] LustreError: 33234:0:(obd_mount_server.c:1844:server_fill_super()) Unable to start targets: -2&lt;br/&gt;
[  861.087424] LustreError: 33234:0:(obd_mount_server.c:1558:server_put_super()) no obd lsdraid-OST0000&lt;br/&gt;
[  861.236785] Lustre: server umount lsdraid-OST0000 complete&lt;br/&gt;
[  861.244448] LustreError: 33234:0:(obd_mount.c:1453:lustre_fill_super()) Unable to mount  (-2)&lt;/p&gt;</comment>
                            <comment id="161982" author="kit.westneat" created="Tue, 16 Aug 2016 04:04:17 +0000"  >&lt;p&gt;I spent some time digging into it. It looks like there&apos;s a slight difference in the way new ldiskfs index files and new ZFS index files work. I&apos;m working on a patch to fix it, but in the meantime I think that workaround I talked about earlier should work if you aren&apos;t using nodemap. Just change the return code from -EINVAL to 0. &lt;/p&gt;</comment>
                            <comment id="161983" author="gerrit" created="Tue, 16 Aug 2016 04:28:36 +0000"  >&lt;p&gt;Kit Westneat (kit.westneat@gmail.com) uploaded a new patch: &lt;a href=&quot;http://review.whamcloud.com/21939&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/21939&lt;/a&gt;&lt;br/&gt;
Subject: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8498&quot; title=&quot;configuration from log &amp;#39;nodemap&amp;#39; failed (-22)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8498&quot;&gt;&lt;del&gt;LU-8498&lt;/del&gt;&lt;/a&gt; nodemap: new zfs index files not properly initialized&lt;br/&gt;
Project: fs/lustre-release&lt;br/&gt;
Branch: master&lt;br/&gt;
Current Patch Set: 1&lt;br/&gt;
Commit: e5097f35594e265803a363f4e2e6fc6ea62f62fc&lt;/p&gt;</comment>
                            <comment id="162134" author="jsalians_intel" created="Tue, 16 Aug 2016 22:10:44 +0000"  >&lt;p&gt;Build #139 (Aug 16, 2016 7:48:23 PM)&lt;br/&gt;
add description&lt;br/&gt;
	Changes  &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8498&quot; title=&quot;configuration from log &amp;#39;nodemap&amp;#39; failed (-22)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8498&quot;&gt;&lt;del&gt;LU-8498&lt;/del&gt;&lt;/a&gt; nodemap: new zfs index files not properly initialized (detail)&lt;br/&gt;
	Manually triggered by user johnsali for Gerrit: &lt;a href=&quot;http://review.whamcloud.com/21939&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/21939&lt;/a&gt; in silent mode.&lt;br/&gt;
Commit 3f2b10c706db51b156bec4516ca277e116663482 by kit.westneat&lt;br/&gt;
&lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8498&quot; title=&quot;configuration from log &amp;#39;nodemap&amp;#39; failed (-22)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8498&quot;&gt;&lt;del&gt;LU-8498&lt;/del&gt;&lt;/a&gt; nodemap: new zfs index files not properly initialized&lt;br/&gt;
Calling index -&amp;gt;next on a new zfs returns a non-zero RC, but ldiskfs &lt;br/&gt;
indexes start with a blank record. This change modifies the config load&lt;br/&gt;
code to always write the default nodemap to an empty index file.&lt;br/&gt;
Signed-off-by: Kit Westneat &amp;lt;kit.westneat@gmail.com&amp;gt; Change-Id:&lt;br/&gt;
I30a365f65463979889f09f7ad5ffcdacc83fa868&lt;/p&gt;

&lt;p&gt;loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t server -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-4 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t server -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-5 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t server -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-3 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-6 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-7 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-8 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 139 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-9 -r &amp;gt;&amp;gt; ljb_log&lt;/p&gt;

&lt;p&gt;Same error for me: &lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zpool status -v ost0&lt;br/&gt;
  pool: ost0&lt;br/&gt;
 state: ONLINE&lt;br/&gt;
  scan: none requested&lt;br/&gt;
config:&lt;/p&gt;

&lt;p&gt;	NAME                      STATE     READ WRITE CKSUM&lt;br/&gt;
	ost0                      ONLINE       0     0     0&lt;br/&gt;
	  draid1-0 &lt;/p&gt;
{any}
&lt;p&gt;          ONLINE       0     0     0&lt;br/&gt;
	    mpathaj               ONLINE       0     0     0&lt;br/&gt;
	    mpathai               ONLINE       0     0     0&lt;br/&gt;
	    mpathah               ONLINE       0     0     0&lt;br/&gt;
	    mpathag               ONLINE       0     0     0&lt;br/&gt;
	    mpathaq               ONLINE       0     0     0&lt;br/&gt;
	    mpathap               ONLINE       0     0     0&lt;br/&gt;
	    mpathak               ONLINE       0     0     0&lt;br/&gt;
	    mpathz                ONLINE       0     0     0&lt;br/&gt;
	    mpatham               ONLINE       0     0     0&lt;br/&gt;
	    mpathal               ONLINE       0     0     0&lt;br/&gt;
	    mpathao               ONLINE       0     0     0&lt;br/&gt;
	  mirror-1 &lt;/p&gt;
{ddt,dmu,mos}
&lt;p&gt;  ONLINE       0     0     0&lt;br/&gt;
	    mpathae               ONLINE       0     0     0&lt;br/&gt;
	    mpathaa               ONLINE       0     0     0&lt;br/&gt;
	spares&lt;br/&gt;
	  $draid1-0-s0            AVAIL   &lt;/p&gt;

&lt;p&gt;errors: No known data errors&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zpool get all ost0 |grep large_blocks&lt;br/&gt;
ost0  feature@large_blocks            enabled                         local&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zfs list &lt;br/&gt;
NAME        USED  AVAIL  REFER  MOUNTPOINT&lt;br/&gt;
ost0       1.35M  7.01T    19K  /ost0&lt;br/&gt;
ost0/ost0  1.21M  7.01T  1.21M  /ost0/ost0&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# mount -t lustre ost0/ost0 /mnt/lustre/ost0&lt;br/&gt;
mount.lustre: mount ost0/ost0 at /mnt/lustre/ost0 failed: No such file or directory&lt;br/&gt;
Is the MGS specification correct?&lt;br/&gt;
Is the filesystem name correct?&lt;br/&gt;
If upgrading, is the copied client log valid? (see upgrade docs)&lt;/p&gt;

&lt;p&gt;[ 4669.245843] LustreError: 13a-8: Failed to get MGS log lsdraid-OST0000 and no local copy.&lt;br/&gt;
[ 4669.255978] LustreError: 15c-8: MGC192.168.1.5@o2ib: The configuration from log &apos;lsdraid-OST0000&apos; failed (-2). This may be the result of communication errors between this node and the MGS, a bad configuration, or other errors. See the syslog for more information.&lt;br/&gt;
[ 4669.284565] LustreError: 48267:0:(obd_mount_server.c:1352:server_start_targets()) failed to start server lsdraid-OST0000: -2&lt;br/&gt;
[ 4669.303204] LustreError: 48267:0:(obd_mount_server.c:1844:server_fill_super()) Unable to start targets: -2&lt;br/&gt;
[ 4669.303994] LustreError: 48267:0:(obd_mount_server.c:1558:server_put_super()) no obd lsdraid-OST0000&lt;br/&gt;
[ 4669.465491] Lustre: server umount lsdraid-OST0000 complete&lt;br/&gt;
[ 4669.472711] LustreError: 48267:0:(obd_mount.c:1453:lustre_fill_super()) Unable to mount  (-2)&lt;/p&gt;</comment>
                            <comment id="162159" author="jsalians_intel" created="Wed, 17 Aug 2016 12:12:54 +0000"  >&lt;p&gt;Looks like the latest patch failed two tests: &lt;/p&gt;

&lt;p&gt; == sanity test 101f: check mmap read performance ===================================================== 15:55:08 (1471388108)&lt;br/&gt;
/usr/bin/iozone&lt;br/&gt;
Cancel LRU locks on lustre client to flush the client cache&lt;br/&gt;
Reset readahead stats&lt;br/&gt;
mmap read the file with small block size&lt;br/&gt;
checking missing pages&lt;br/&gt;
 sanity test_101f: @@@@@@ FAIL: misses too much pages! &lt;br/&gt;
  Trace dump:&lt;br/&gt;
  = /usr/lib64/lustre/tests/test-framework.sh:4817:error()&lt;br/&gt;
  = /usr/lib64/lustre/tests/sanity.sh:6671:test_101f()&lt;br/&gt;
  = /usr/lib64/lustre/tests/test-framework.sh:5081:run_one()&lt;br/&gt;
  = /usr/lib64/lustre/tests/test-framework.sh:5120:run_one_logged()&lt;br/&gt;
  = /usr/lib64/lustre/tests/test-framework.sh:4967:run_test()&lt;br/&gt;
  = /usr/lib64/lustre/tests/sanity.sh:6674:main()&lt;/p&gt;

&lt;p&gt;Here is the debug log: &lt;a href=&quot;https://testing.hpdd.intel.com/test_logs/de8292ec-6437-11e6-906c-5254006e85c2/show_text&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://testing.hpdd.intel.com/test_logs/de8292ec-6437-11e6-906c-5254006e85c2/show_text&lt;/a&gt; &lt;br/&gt;
...etc...&lt;br/&gt;
00000080:00200000:1.0:1471388110.672492:0:19130:0:(rw26.c:779:ll_write_end()) queued page: 255.&lt;br/&gt;
00000080:00200000:1.0:1471388110.672493:0:19130:0:(rw26.c:647:ll_write_begin()) Writing 32767 of 0 to 4096 bytes&lt;br/&gt;
00000080:00200000:1.0:1471388110.672496:0:19130:0:(rw26.c:779:ll_write_end()) page@ffff8805daa07a00&lt;span class=&quot;error&quot;&gt;&amp;#91;3 ffff8805fbf98740 1 1 ffff8805fd1a1b78&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;00000080:00200000:1.0:1471388110.672497:0:19130:0:(rw26.c:779:ll_write_end()) vvp-page@ffff8805daa07a50(0:0) vm@ffffea00179166c0 2fffff00000805 3:0 ffff8805daa07a00 32767 lru&lt;/p&gt;

&lt;p&gt;00000080:00200000:1.0:1471388110.672498:0:19130:0:(rw26.c:779:ll_write_end()) lov-page@ffff8805daa07a90, raid0&lt;/p&gt;

&lt;p&gt;00000080:00200000:1.0:1471388110.672501:0:19130:0:(rw26.c:779:ll_write_end()) osc-page@ffff8805daa07af8 32767: 1&amp;lt; 0x845fed 0 0 - - &amp;gt; 2&amp;lt; 134213632 0 0 0x0 0x100 |           (null) ffff8805fda93270 ffff8805fbec04c0 &amp;gt; 3&amp;lt; 0 0 0 &amp;gt; 4&amp;lt; 0 0 8 33816576 - | - - - - &amp;gt; 5&amp;lt; - - - - | 0 - | 0 - -&amp;gt;&lt;/p&gt;

&lt;p&gt;00000080:00200000:1.0:1471388110.672502:0:19130:0:(rw26.c:779:ll_write_end()) end page@ffff8805daa07a00&lt;/p&gt;

&lt;p&gt;00000080:00200000:1.0:1471388110.672502:0:19130:0:(rw26.c:779:ll_write_end()) queued page: 256.&lt;br/&gt;
00000080:00200000:1.0:1471388110.672503:0:19130:0:(vvp_io.c:902:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096&lt;br/&gt;
00000008:00010000:1.0:1471388110.672509:0:19130:0:(osc_cache.c:816:osc_extent_find()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672528:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672545:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672561:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672578:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672594:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672610:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:1.0:1471388110.672626:0:19130:0:(osc_cache.c:2395:osc_queue_async_io()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000080:00200000:1.0:1471388110.672643:0:19130:0:(vvp_io.c:924:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576&lt;br/&gt;
00000080:00200000:1.0:1471388110.672644:0:19130:0:(file.c:1036:ll_merge_attr()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; updating i_size 134217728&lt;br/&gt;
00000080:00200000:1.0:1471388110.672645:0:19130:0:(vvp_io.c:1042:vvp_io_write_start()) write: nob 1048576, result: 1048576&lt;br/&gt;
00000020:00010000:1.0:1471388110.672650:0:19130:0:(cl_lock.c:58:cl_lock_trace0()) release lock: ffff880617cc4eb8 (ffff8805fd246998/0) at cl_lock_release():237&lt;br/&gt;
00000020:00010000:1.0:1471388110.672651:0:19130:0:(cl_lock.c:58:cl_lock_trace0()) cancel lock: ffff880617cc4eb8 (ffff8805fd246998/0) at cl_lock_cancel():154&lt;br/&gt;
00000100:00100000:0.0:1471388110.672652:0:6896:0:(ptlrpcd.c:411:ptlrpcd_check()) transfer 1 async RPCs &lt;span class=&quot;error&quot;&gt;&amp;#91;1-&amp;gt;0&amp;#93;&lt;/span&gt;&lt;br/&gt;
00000020:00010000:1.0:1471388110.672653:0:19130:0:(cl_lock.c:58:cl_lock_trace0()) cancel lock: ffff880619f20a30 (ffff880608208758/1) at cl_lock_cancel():154&lt;br/&gt;
00010000:00010000:1.0:1471388110.672654:0:19130:0:(ldlm_lock.c:800:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 5/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x10020000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:0.0:1471388110.672655:0:6896:0:(osc_cache.c:1097:osc_extent_make_ready()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 5/0,1 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x10020000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00010000:00010000:1.0:1471388110.672657:0:19130:0:(ldlm_lock.c:873:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 4/0,0 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x10020000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000020:00010000:1.0:1471388110.672660:0:19130:0:(cl_lock.c:58:cl_lock_trace0()) destroy lock: ffff880617cc4eb8 (ffff8805fd246998/0) at cl_lock_fini():89&lt;br/&gt;
00000020:00010000:1.0:1471388110.672661:0:19130:0:(cl_lock.c:58:cl_lock_trace0()) destroy lock: ffff880619f20a30 (ffff8805fd246998/1) at cl_lock_fini():89&lt;br/&gt;
00000080:00200000:1.0:1471388110.672663:0:19130:0:(file.c:1169:ll_file_io_generic()) Range unlock &lt;span class=&quot;error&quot;&gt;&amp;#91;32512, 32767&amp;#93;&lt;/span&gt;&lt;br/&gt;
00000080:00200000:1.0:1471388110.672664:0:19130:0:(vvp_io.c:312:vvp_io_fini()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 0/0, layout version 0 restore needed 0&lt;br/&gt;
00000080:00200000:1.0:1471388110.672665:0:19130:0:(file.c:1219:ll_file_io_generic()) iot: 1, result: 1048576&lt;br/&gt;
00000080:00200000:1.0:1471388110.672671:0:19130:0:(file.c:2933:ll_fsync()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;(ffff8805fbddda90)&lt;br/&gt;
00000080:00200000:1.0:1471388110.672674:0:19130:0:(vvp_io.c:1373:vvp_io_init()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 0/0, layout version 0 restore needed 0&lt;br/&gt;
00020000:00200000:1.0:1471388110.672677:0:19130:0:(lov_io.c:417:lov_io_iter_init()) shrink: 0 [0, 18446744073709551615)&lt;br/&gt;
00010000:00010000:0.0:1471388110.672740:0:6896:0:(ldlm_lock.c:1474:ldlm_lock_match()) ### matched (133169152 133173247) ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 3/0,0 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00100000:0.0:1471388110.672866:0:6896:0:(osc_request.c:1315:osc_brw_prep_request()) brw rpc ffff8805f6020900 - object 0x0:6437 offset 133169152&amp;lt;&amp;gt;134217728&lt;br/&gt;
00000100:00100000:2.0:1471388110.672906:0:6898:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc ptlrpcd_00_02:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:6898:1542858500576320:192.168.5.153@o2ib:4&lt;br/&gt;
00000008:00010000:2.0:1471388110.674148:0:6898:0:(osc_cache.c:857:osc_extent_finish()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 2/0,0 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000008:00010000:2.0:1471388110.674232:0:6898:0:(osc_cache.c:489:__osc_extent_remove()) ### extent: ffff8805f7e7cfc0 ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 2/0,0 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x20000000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000100:00100000:2.0:1471388110.674239:0:6898:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc ptlrpcd_00_02:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:6898:1542858500576320:192.168.5.153@o2ib:4&lt;br/&gt;
00000080:00200000:1.0:1471388110.674265:0:19130:0:(vvp_io.c:312:vvp_io_fini()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 0/0, layout version 0 restore needed 0&lt;br/&gt;
00000100:00100000:1.0:1471388110.674286:0:19130:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19130:1542858500576336:192.168.5.144@o2ib:44&lt;br/&gt;
00000100:00100000:1.0:1471388110.674303:0:19130:0:(client.c:2333:ptlrpc_set_wait()) set ffff88061bd7a900 going to sleep for 6 seconds&lt;br/&gt;
00000100:00100000:1.0:1471388110.917462:0:19130:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19130:1542858500576336:192.168.5.144@o2ib:44&lt;br/&gt;
00000080:00200000:1.0:1471388110.917471:0:19130:0:(vvp_io.c:1373:vvp_io_init()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 0/0, layout version 0 restore needed 0&lt;br/&gt;
00020000:00200000:1.0:1471388110.917475:0:19130:0:(lov_io.c:417:lov_io_iter_init()) shrink: 0 [0, 9223372036854775807)&lt;br/&gt;
00000100:00100000:2.0:1471388110.917514:0:6898:0:(ptlrpcd.c:411:ptlrpcd_check()) transfer 1 async RPCs &lt;span class=&quot;error&quot;&gt;&amp;#91;3-&amp;gt;2&amp;#93;&lt;/span&gt;&lt;br/&gt;
00000100:00100000:2.0:1471388110.917520:0:6898:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc ptlrpcd_00_02:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:6898:1542858500576352:192.168.5.153@o2ib:16&lt;br/&gt;
00000100:00100000:2.0:1471388112.175215:0:6898:0:(client.c:2687:ptlrpc_free_committed()) lustre-OST0001-osc-ffff880bf93c0000: committing for last_committed 21474841962 gen 1&lt;br/&gt;
00000100:00100000:2.0:1471388112.176550:0:6898:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc ptlrpcd_00_02:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:6898:1542858500576352:192.168.5.153@o2ib:16&lt;br/&gt;
00000080:00200000:1.0:1471388112.176597:0:19130:0:(vvp_io.c:312:vvp_io_fini()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 0/0, layout version 0 restore needed 0&lt;br/&gt;
00000080:00200000:1.0:1471388112.176613:0:19130:0:(file.c:323:ll_file_release()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;(ffff8805fbddda90)&lt;br/&gt;
00000002:00080000:1.0:1471388112.176641:0:19130:0:(mdc_request.c:782:mdc_close()) @@@ matched open  req@ffff8800b4abb900 x1542858500572144/t21474859659(21474859659) o101-&amp;gt;lustre-MDT0000-mdc-ffff880bf93c0000@192.168.5.144@o2ib:12/10 lens 816/600 e 0 to 0 dl 1471388117 ref 1 fl Complete:RP/4/0 rc 0/0&lt;br/&gt;
00000100:00100000:1.0:1471388112.176666:0:19130:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19130:1542858500576368:192.168.5.144@o2ib:35&lt;br/&gt;
00000100:00100000:1.0:1471388112.176700:0:19130:0:(client.c:2333:ptlrpc_set_wait()) set ffff88061bd7a900 going to sleep for 6 seconds&lt;br/&gt;
00000100:00100000:1.0:1471388112.177257:0:19130:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19130:1542858500576368:192.168.5.144@o2ib:35&lt;br/&gt;
00000002:00100000:1.0:1471388112.177274:0:19130:0:(mdc_request.c:710:mdc_free_open()) @@@ free open request rq_replay= 0&lt;br/&gt;
  req@ffff8800b4abb900 x1542858500572144/t21474859659(21474859659) o101-&amp;gt;lustre-MDT0000-mdc-ffff880bf93c0000@192.168.5.144@o2ib:12/10 lens 816/600 e 0 to 0 dl 1471388117 ref 1 fl Complete:R/4/0 rc 0/0&lt;br/&gt;
00010000:00010000:17.0:1471388112.181769:0:19132:0:(ldlm_resource.c:323:lprocfs_lru_size_seq_write()) dropping all unused locks from namespace lustre-OST0000-osc-ffff880bf93c0000&lt;br/&gt;
00010000:00010000:17.0:1471388112.181823:0:19132:0:(ldlm_resource.c:323:lprocfs_lru_size_seq_write()) dropping all unused locks from namespace lustre-OST0001-osc-ffff880bf93c0000&lt;br/&gt;
00010000:00010000:7.0:1471388112.181886:0:13815:0:(ldlm_request.c:1115:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 2/0,0 mode: PW/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0-&amp;gt;1048575) flags: 0x28400000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00000080:00200000:7.0:1471388112.181908:0:13815:0:(vvp_io.c:1373:vvp_io_init()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 1/0, layout version 0 restore needed 0&lt;br/&gt;
00000080:00200000:7.0:1471388112.242363:0:13815:0:(vvp_io.c:312:vvp_io_fini()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 1/0, layout version 0 restore needed 0&lt;br/&gt;
00010000:00010000:7.0:1471388112.242395:0:13815:0:(ldlm_request.c:1174:ldlm_cancel_pack()) ### packing ns: lustre-OST0001-osc-ffff880bf93c0000 lock: ffff8805fc0c3e00/0x96bbf1af18b6cddd lrc: 1/0,0 mode: -&lt;del&gt;/PW res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1925:0x0:0x0&amp;#93;&lt;/span&gt;.0x0 rrc: 1 type: EXT &lt;span class=&quot;error&quot;&gt;&amp;#91;0-&amp;gt;18446744073709551615&amp;#93;&lt;/span&gt; (req 0&lt;/del&gt;&amp;gt;1048575) flags: 0x4869400000000 nid: local remote: 0x90fceb1b494878f5 expref: -99 pid: 19130 timeout: 0 lvb_type: 1&lt;br/&gt;
00010000:00010000:7.0:1471388112.242402:0:13815:0:(ldlm_request.c:1178:ldlm_cancel_pack()) 1 locks packed&lt;br/&gt;
00000100:00100000:7.0:1471388112.242414:0:13815:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc ldlm_bl_13:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:13815:1542858500576384:192.168.5.153@o2ib:103&lt;br/&gt;
00000100:00100000:7.0:1471388112.242451:0:13815:0:(client.c:2333:ptlrpc_set_wait()) set ffff8805fd730e40 going to sleep for 6 seconds&lt;br/&gt;
00000100:00100000:7.0:1471388112.242942:0:13815:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc ldlm_bl_13:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:13815:1542858500576384:192.168.5.153@o2ib:103&lt;br/&gt;
...etc...&lt;br/&gt;
00000020:00010000:6.0:1471388113.593184:0:19136:0:(cl_lock.c:58:cl_lock_trace0()) destroy lock: ffff880c1bbe8030 (ffff8806082087e8/1) at cl_lock_fini():89&lt;br/&gt;
00000080:00200000:6.0:1471388113.593188:0:19136:0:(vvp_io.c:312:vvp_io_fini()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt; ignore/verify layout 0/1, layout version 0 restore needed 0&lt;br/&gt;
00000080:00200000:6.0:1471388113.593207:0:19136:0:(file.c:3410:__ll_inode_revalidate()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410),name=/&lt;br/&gt;
00000002:00010000:6.0:1471388113.593212:0:19136:0:(mdc_locks.c:1083:mdc_intent_lock()) (name: ,&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;) in obj &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;, intent: lookup flags 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.593216:0:19136:0:(ldlm_lock.c:746:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.593225:0:19136:0:(ldlm_lock.c:1474:ldlm_lock_match()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x0 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00000080:00010000:6.0:1471388113.593230:0:19136:0:(dcache.c:298:ll_lookup_finish_locks()) setting l_data to inode &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410)&lt;br/&gt;
00000080:00010000:6.0:1471388113.593232:0:19136:0:(llite_internal.h:1339:ll_set_lock_data()) setting l_data to inode &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410) for lock 0x96bbf1af18b6a255&lt;br/&gt;
00000080:00010000:6.0:1471388113.593234:0:19136:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x96bbf1af18b6a255 from it ffff880bf787bbd8&lt;br/&gt;
00010000:00010000:6.0:1471388113.593235:0:19136:0:(ldlm_lock.c:800:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 3/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x10000000000000 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.593239:0:19136:0:(ldlm_lock.c:873:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/0,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x10000000000000 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00000080:00200000:6.0:1471388113.593243:0:19136:0:(file.c:3708:ll_inode_permission()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410), inode mode 41ed mask 1&lt;br/&gt;
00000080:00200000:6.0:1471388113.593248:0:19136:0:(dcache.c:361:ll_revalidate_nd()) VFS Op:name=f101f.sanity, flags=0&lt;br/&gt;
00000080:00200000:6.0:1471388113.593250:0:19136:0:(file.c:3410:__ll_inode_revalidate()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410),name=/&lt;br/&gt;
00000002:00010000:6.0:1471388113.593252:0:19136:0:(mdc_locks.c:1083:mdc_intent_lock()) (name: ,&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;) in obj &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;, intent: lookup flags 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.593254:0:19136:0:(ldlm_lock.c:746:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.593258:0:19136:0:(ldlm_lock.c:1474:ldlm_lock_match()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x0 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00000080:00010000:6.0:1471388113.593263:0:19136:0:(dcache.c:298:ll_lookup_finish_locks()) setting l_data to inode &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410)&lt;br/&gt;
00000080:00010000:6.0:1471388113.593264:0:19136:0:(llite_internal.h:1339:ll_set_lock_data()) setting l_data to inode &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410) for lock 0x96bbf1af18b6a255&lt;br/&gt;
00000080:00010000:6.0:1471388113.593266:0:19136:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x96bbf1af18b6a255 from it ffff880bf787bd18&lt;br/&gt;
00010000:00010000:6.0:1471388113.593267:0:19136:0:(ldlm_lock.c:800:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 3/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x10000000000000 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.593275:0:19136:0:(ldlm_lock.c:873:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/0,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x10000000000000 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00000080:00200000:6.0:1471388113.593279:0:19136:0:(file.c:3708:ll_inode_permission()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410), inode mode 41ed mask 3&lt;br/&gt;
00000080:00200000:6.0:1471388113.593284:0:19136:0:(namei.c:1323:ll_unlink()) VFS Op:name=f101f.sanity, dir=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410)&lt;br/&gt;
00010000:00010000:6.0:1471388113.593293:0:19136:0:(ldlm_request.c:1115:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5a9000/0x96bbf1af18b6cdf9 lrc: 2/0,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x1b rrc: 3 type: IBT flags: 0x28400000000 nid: local remote: 0x1a97a8993567018b expref: -99 pid: 19136 timeout: 0 lvb_type: 3&lt;br/&gt;
00000080:00200000:6.0:1471388113.593304:0:19136:0:(vvp_object.c:141:vvp_conf_set()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;: losing layout lock&lt;br/&gt;
00010000:00010000:6.0:1471388113.593310:0:19136:0:(ldlm_request.c:1115:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fc0c4200/0x96bbf1af18b6cdcf lrc: 2/0,0 mode: CR/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x9 rrc: 3 type: IBT flags: 0x28400000000 nid: local remote: 0x1a97a89935670137 expref: -99 pid: 19130 timeout: 0 lvb_type: 3&lt;br/&gt;
00000080:00200000:6.0:1471388113.593315:0:19136:0:(vvp_object.c:141:vvp_conf_set()) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;: losing layout lock&lt;br/&gt;
00010000:00010000:6.0:1471388113.593324:0:19136:0:(ldlm_request.c:1174:ldlm_cancel_pack()) ### packing ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5a9000/0x96bbf1af18b6cdf9 lrc: 1/0,0 mode: --/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x1b rrc: 2 type: IBT flags: 0x4829400000000 nid: local remote: 0x1a97a8993567018b expref: -99 pid: 19136 timeout: 0 lvb_type: 3&lt;br/&gt;
00010000:00010000:6.0:1471388113.593329:0:19136:0:(ldlm_request.c:1174:ldlm_cancel_pack()) ### packing ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fc0c4200/0x96bbf1af18b6cdcf lrc: 1/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x9 rrc: 2 type: IBT flags: 0x4829400000000 nid: local remote: 0x1a97a89935670137 expref: -99 pid: 19130 timeout: 0 lvb_type: 3&lt;br/&gt;
00010000:00010000:6.0:1471388113.593334:0:19136:0:(ldlm_request.c:1178:ldlm_cancel_pack()) 2 locks packed&lt;br/&gt;
00010000:00010000:6.0:1471388113.593335:0:19136:0:(ldlm_lock.c:200:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5a9000/0x96bbf1af18b6cdf9 lrc: 0/0,0 mode: --/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x1b rrc: 2 type: IBT flags: 0x4829400000000 nid: local remote: 0x1a97a8993567018b expref: -99 pid: 19136 timeout: 0 lvb_type: 3&lt;br/&gt;
00010000:00010000:6.0:1471388113.593340:0:19136:0:(ldlm_lock.c:200:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fc0c4200/0x96bbf1af18b6cdcf lrc: 0/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x2000013a0:0xe9:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x9 rrc: 1 type: IBT flags: 0x4829400000000 nid: local remote: 0x1a97a89935670137 expref: -99 pid: 19130 timeout: 0 lvb_type: 3&lt;br/&gt;
00000100:00100000:6.0:1471388113.593354:0:19136:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19136:1542858500609344:192.168.5.144@o2ib:36&lt;br/&gt;
00000100:00100000:6.0:1471388113.593373:0:19136:0:(client.c:2333:ptlrpc_set_wait()) set ffff8805f6d15c80 going to sleep for 6 seconds&lt;br/&gt;
00000100:00100000:6.0:1471388113.594796:0:19136:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19136:1542858500609344:192.168.5.144@o2ib:36&lt;br/&gt;
00000080:00200000:6.0:1471388113.594832:0:19136:0:(file.c:3410:__ll_inode_revalidate()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410),name=/&lt;br/&gt;
00000002:00010000:6.0:1471388113.594836:0:19136:0:(mdc_locks.c:1083:mdc_intent_lock()) (name: ,&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;) in obj &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;, intent: lookup flags 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.594840:0:19136:0:(ldlm_lock.c:746:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.594845:0:19136:0:(ldlm_lock.c:1474:ldlm_lock_match()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x0 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00000080:00010000:6.0:1471388113.594850:0:19136:0:(dcache.c:298:ll_lookup_finish_locks()) setting l_data to inode &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410)&lt;br/&gt;
00000080:00010000:6.0:1471388113.594851:0:19136:0:(llite_internal.h:1339:ll_set_lock_data()) setting l_data to inode &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410) for lock 0x96bbf1af18b6a255&lt;br/&gt;
00000080:00010000:6.0:1471388113.594853:0:19136:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x96bbf1af18b6a255 from it ffff880bf787bb40&lt;br/&gt;
00010000:00010000:6.0:1471388113.594854:0:19136:0:(ldlm_lock.c:800:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 3/1,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x10000000000000 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.594859:0:19136:0:(ldlm_lock.c:873:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff880bfae55e00/0x96bbf1af18b6a255 lrc: 2/0,0 mode: PR/PR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x11 rrc: 1 type: IBT flags: 0x10000000000000 nid: local remote: 0x1a97a8993566831b expref: -99 pid: 5162 timeout: 0 lvb_type: 0&lt;br/&gt;
00000080:00200000:6.0:1471388113.594863:0:19136:0:(file.c:3708:ll_inode_permission()) VFS Op:inode=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410), inode mode 41ed mask 1&lt;br/&gt;
00000080:00200000:6.0:1471388113.594868:0:19136:0:(namei.c:665:ll_lookup_nd()) VFS Op:name=f101f.sanity.DUMMY, dir=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410), flags=1&lt;br/&gt;
00000080:00200000:6.0:1471388113.594871:0:19136:0:(namei.c:559:ll_lookup_it()) VFS Op:name=f101f.sanity.DUMMY, dir=&lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;(ffff880bf55fc410), intent=getattr&lt;br/&gt;
00000002:00010000:6.0:1471388113.594875:0:19136:0:(mdc_locks.c:1083:mdc_intent_lock()) (name: f101f.sanity.DUMMY,&lt;span class=&quot;error&quot;&gt;&amp;#91;0x0:0x0:0x0&amp;#93;&lt;/span&gt;) in obj &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;, intent: getattr flags 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.594886:0:19136:0:(ldlm_lock.c:746:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 3/1,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x0 rrc: 2 type: IBT flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.594890:0:19136:0:(ldlm_request.c:921:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 3/1,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.594895:0:19136:0:(ldlm_request.c:976:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 3/1,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00000100:00100000:6.0:1471388113.594906:0:19136:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19136:1542858500609360:192.168.5.144@o2ib:101&lt;br/&gt;
00000100:00100000:6.0:1471388113.594918:0:19136:0:(client.c:2333:ptlrpc_set_wait()) set ffff8805f6d15c80 going to sleep for 6 seconds&lt;br/&gt;
00000100:00100000:6.0:1471388113.595565:0:19136:0:(client.c:2043:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc iozone:8db7504e-a9aa-cce3-6ec0-a9745ceb5f0c:19136:1542858500609360:192.168.5.144@o2ib:101&lt;br/&gt;
00010000:00010000:6.0:1471388113.595586:0:19136:0:(ldlm_request.c:567:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 4/1,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595595:0:19136:0:(ldlm_request.c:513:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 4/1,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595600:0:19136:0:(ldlm_lock.c:800:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 4/1,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595605:0:19136:0:(ldlm_lock.c:856:ldlm_lock_decref_internal()) ### final decref done on cbpending lock ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 3/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595610:0:19136:0:(ldlm_lockd.c:1749:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 4/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595614:0:19136:0:(ldlm_lockd.c:1762:ldlm_handle_bl_callback()) Lock ffff8805fd5aa200 already unused, calling callback (ffffffffa0eceff0)&lt;br/&gt;
00010000:00010000:6.0:1471388113.595616:0:19136:0:(ldlm_request.c:1115:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 5/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595628:0:19136:0:(ldlm_request.c:1128:ldlm_cli_cancel_local()) not sending request (at caller&apos;s instruction)&lt;br/&gt;
00010000:00010000:6.0:1471388113.595630:0:19136:0:(ldlm_lockd.c:1771:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 3/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00010000:00010000:6.0:1471388113.595635:0:19136:0:(ldlm_lock.c:200:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff880bf93c0000 lock: ffff8805fd5aa200/0x96bbf1af18b6ce00 lrc: 0/0,0 mode: --/CR res: &lt;span class=&quot;error&quot;&gt;&amp;#91;0x200000007:0x1:0x0&amp;#93;&lt;/span&gt;.0x0 bits 0x2 rrc: 2 type: IBT flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 19136 timeout: 0 lvb_type: 0&lt;br/&gt;
00000002:00100000:6.0:1471388113.595646:0:19136:0:(mdc_locks.c:587:mdc_finish_enqueue()) @@@ op: 8 disposition: 7, status: 0  req@ffff8805f8e1b300 x1542858500609360/t0(0) o101-&amp;gt;lustre-MDT0000-mdc-ffff880bf93c0000@192.168.5.144@o2ib:12/10 lens 712/536 e 0 to 0 dl 1471388120 ref 1 fl Complete:R/0/0 rc 301/301&lt;br/&gt;
00010000:00010000:6.0:1471388113.595659:0:19136:0:(ldlm_lock.c:1495:ldlm_lock_match()) ### not matched ns ffff8805fa8cb000 type 13 mode 30 res 8589934599/1 (0 0)&lt;br/&gt;
This looks like &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8413&quot; title=&quot;sanity test_101f fails with &amp;#39;misses too much pages!&amp;#39; &quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8413&quot;&gt;&lt;del&gt;LU-8413&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;and test_21 which had a TIMEOUT and test failed to respond and it timed out.  It appeared the OSS node ran out of memory during the test which caused processes including system.journal to be killed.  It appears that this caused the dd on the client to be continually blocked for me than 120 seconds and never make progress.  18:57:54:[   35.401655] Mem-Info:&lt;br/&gt;
18:57:54:[   35.405029] Node 0 DMA per-cpu:&lt;br/&gt;
18:57:54:[   35.409384] CPU    0: hi:    0, btch:   1 usd:   0&lt;br/&gt;
18:57:54:[   35.415581] Node 0 DMA32 per-cpu:&lt;br/&gt;
18:57:54:[   35.420128] CPU    0: hi:   42, btch:   7 usd:  36&lt;br/&gt;
18:57:54:[   35.426323] active_anon:632 inactive_anon:2092 isolated_anon:0&lt;br/&gt;
18:57:54:[   35.426323]  active_&lt;a href=&quot;file:0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0&lt;/a&gt; inactive_&lt;a href=&quot;file:0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0&lt;/a&gt; isolated_&lt;a href=&quot;file:0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0&lt;/a&gt;&lt;br/&gt;
18:57:54:[   35.426323]  unevictable:18297 dirty:0 writeback:0 unstable:0&lt;br/&gt;
18:57:54:[   35.426323]  free:499 slab_reclaimable:1742 slab_unreclaimable:4599&lt;br/&gt;
18:57:54:[   35.426323]  mapped:748 shmem:2129 pagetables:74 bounce:0&lt;br/&gt;
18:57:54:[   35.426323]  free_cma:0&lt;br/&gt;
18:57:54:[   35.466224] Node 0 DMA free:488kB min:4kB low:4kB high:4kB active_anon:0kB inactive_anon:0kB active_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; inactive_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; unevictable:0kB isolated(anon):0kB isolated(file):0kB present:572kB managed:488kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes&lt;br/&gt;
18:57:54:[   35.516862] lowmem_reserve[]: 0 140 140 140&lt;br/&gt;
18:57:54:[   35.523578] Node 0 DMA32 free:1508kB min:1512kB low:1888kB high:2268kB active_anon:2528kB inactive_anon:8368kB active_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; inactive_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; unevictable:73188kB isolated(anon):0kB isolated(file):0kB present:166320kB managed:145724kB mlocked:0kB dirty:0kB writeback:0kB mapped:2992kB shmem:8516kB slab_reclaimable:6968kB slab_unreclaimable:18396kB kernel_stack:1344kB pagetables:296kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes&lt;br/&gt;
18:57:54:[   35.579210] lowmem_reserve[]: 0 0 0 0&lt;br/&gt;
18:57:54:[   35.585507] Node 0 DMA: 0*4kB 1*8kB (U) 0*16kB 1*32kB (U) 1*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 488kB&lt;br/&gt;
18:57:54:[   35.603198] Node 0 DMA32: 183*4kB (UEMR) 49*8kB (UR) 14*16kB (R) 5*32kB (R) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 1508kB&lt;br/&gt;
18:57:54:[   35.621271] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB&lt;br/&gt;
18:57:54:[   35.632651] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB&lt;br/&gt;
18:57:54:[   35.643597] 20426 total pagecache pages&lt;br/&gt;
18:57:54:[   35.649313] 0 pages in swap cache&lt;br/&gt;
18:57:54:[   35.654428] Swap cache stats: add 0, delete 0, find 0/0&lt;br/&gt;
18:57:54:[   35.661603] Free swap  = 0kB&lt;br/&gt;
18:57:54:[   35.666143] Total swap = 0kB&lt;br/&gt;
18:57:54:[   35.670610] 41723 pages RAM&lt;br/&gt;
18:57:54:[   35.674975] 0 pages HighMem/MovableOnly&lt;br/&gt;
18:57:54:[   35.680511] 5170 pages reserved&lt;br/&gt;
18:57:54:[   35.685199] [ pid ]   uid  tgid total_vm      rss nr_ptes swapents oom_score_adj name&lt;br/&gt;
18:57:54:[   35.695151] [  200]     0   200     4289       99      12        0             0 rpc.idmapd&lt;br/&gt;
18:57:54:[   35.705657] [  227]     0   227     8998      639      21        0         -1000 systemd-udevd&lt;br/&gt;
18:57:54:[   35.716393] [  399]     0   399     7157      316      17        0             0 systemd-journal&lt;br/&gt;
18:57:54:[   35.727277] Out of memory: Kill process 399 (systemd-journal) score 8 or sacrifice child&lt;br/&gt;
18:57:54:[   35.737359] Killed process 399 (systemd-journal) total-vm:28628kB, anon-rss:228kB, file-rss:1036kB&lt;br/&gt;
18:57:54:[   35.771271] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: systemd-journald.service: main process exited, code=killed, status=9/KILL&lt;br/&gt;
18:57:54:[�[1;31mFAILED�&lt;span class=&quot;error&quot;&gt;&amp;#91;0m&amp;#93;&lt;/span&gt; Failed to start Journal Service.&lt;br/&gt;
18:57:54:See &apos;systemctl status systemd-journald.service&apos; for details.&lt;br/&gt;
18:57:54:[   35.796637] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Failed to start Journal Service.&lt;br/&gt;
18:57:54:[   35.803704] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Unit systemd-journald.service entered failed state.&lt;br/&gt;
18:57:54:[   35.812630] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: systemd-journald.service failed.&lt;br/&gt;
18:57:54:[   35.820624] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: systemd-journald.service has no holdoff time, scheduling restart.&lt;br/&gt;
18:57:54:[�[32m  OK  �&lt;span class=&quot;error&quot;&gt;&amp;#91;0m&amp;#93;&lt;/span&gt; Stopped Journal Service.&lt;br/&gt;
18:57:54:[   35.836694] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: start request repeated too quickly for systemd-journald.service&lt;br/&gt;
18:57:54:[�[1;31mFAILED�&lt;span class=&quot;error&quot;&gt;&amp;#91;0m&amp;#93;&lt;/span&gt; Failed to start Journal Service.&lt;br/&gt;
18:57:54:See &apos;systemctl status systemd-journald.service&apos; for details.&lt;br/&gt;
18:57:54:[   35.860669] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Failed to start Journal Service.&lt;br/&gt;
18:57:54:[   35.889869] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Unit systemd-journald.service entered failed state.&lt;br/&gt;
18:57:54:[   35.898766] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: systemd-journald.service failed.&lt;br/&gt;
18:57:54:[   35.905807] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: start request repeated too quickly for systemd-journald.service&lt;br/&gt;
18:57:54:[�[1;31mFAILED�&lt;span class=&quot;error&quot;&gt;&amp;#91;0m&amp;#93;&lt;/span&gt; Failed to start Journal Service.&lt;br/&gt;
18:57:54:See &apos;systemctl status systemd-journald.service&apos; for details.&lt;br/&gt;
18:57:54:[   35.930716] systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Failed to start Journal Service.&lt;br/&gt;
18:57:54:[   35.937928] systemd-udevd&lt;span class=&quot;error&quot;&gt;&amp;#91;227&amp;#93;&lt;/span&gt;: worker &lt;span class=&quot;error&quot;&gt;&amp;#91;384&amp;#93;&lt;/span&gt; failed while handling &apos;/devices/pci0000:00/0000:00:01.0/0000:01:00.0/0000:02:08.0/0000:03:00.0/host0/port-0:0/end_device-0:0/target0:0:0/0:0:0:0/block/sda/sda2&apos; &lt;br/&gt;
This looks like &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8326&quot; title=&quot;sanity-quota test_21 test failed to respond and timed out&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8326&quot;&gt;&lt;del&gt;LU-8326&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Will get a build going and see how it looks on my end.&lt;/p&gt;</comment>
                            <comment id="162192" author="jsalians_intel" created="Wed, 17 Aug 2016 15:46:37 +0000"  >&lt;p&gt;Build #148 (Aug 17, 2016 1:01:09 PM)&lt;/p&gt;

&lt;p&gt;add description&lt;br/&gt;
	Changes &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8498&quot; title=&quot;configuration from log &amp;#39;nodemap&amp;#39; failed (-22)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8498&quot;&gt;&lt;del&gt;LU-8498&lt;/del&gt;&lt;/a&gt; nodemap: new zfs index files not properly initialized (detail)&lt;br/&gt;
Manually triggered by user johnsali for Gerrit: &lt;a href=&quot;http://review.whamcloud.com/21939&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/21939&lt;/a&gt; in silent mode&lt;br/&gt;
Revision: 6ddcac0c807ffa7c847aef5faf46b6643d0ae538&lt;/p&gt;

&lt;p&gt;$ cat reprovision.sh&lt;br/&gt;
date &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t server -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-4 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t server -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-5 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t server -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-3 -r &amp;gt;&amp;gt; ljb_log&lt;/p&gt;

&lt;p&gt;loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-6 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-7 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-8 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
loadjenkinsbuild -b 148 -j zfs-c-p-lustre-patch-vanilla -t client -p test-el7.2-x86_64 -d el7.2 -a x86_64 -v -n wolf-9 -r &amp;gt;&amp;gt; ljb_log&lt;br/&gt;
$ ./reprovision.sh &lt;/p&gt;


&lt;p&gt;MDS 0&lt;br/&gt;
Restarting network (via systemctl):  Job for network.service failed because the control process exited with error code. See &quot;systemctl status network.service&quot; and &quot;journalctl -xe&quot; for details.&lt;/p&gt;

&lt;p&gt;   Permanent disk data:&lt;br/&gt;
Target:     lsdraid:MDT0000&lt;br/&gt;
Index:      0&lt;br/&gt;
Lustre FS:  lsdraid&lt;br/&gt;
Mount type: zfs&lt;br/&gt;
Flags:      0x65&lt;br/&gt;
              (MDT MGS first_time update )&lt;br/&gt;
Persistent mount opts: &lt;br/&gt;
Parameters:&lt;/p&gt;

&lt;p&gt;checking for existing Lustre data: not found&lt;br/&gt;
mkfs_cmd = zpool create -f -O canmount=off zfspool mirror /dev/sdb /dev/sdc mirror /dev/sdd /dev/sde&lt;br/&gt;
mkfs_cmd = zfs create -o canmount=off -o xattr=sa zfspool/mdt1&lt;br/&gt;
Writing zfspool/mdt1 properties&lt;br/&gt;
  lustre:version=1&lt;br/&gt;
  lustre:flags=101&lt;br/&gt;
  lustre:index=0&lt;br/&gt;
  lustre:fsname=lsdraid&lt;br/&gt;
  lustre:svname=lsdraid:MDT0000&lt;/p&gt;

&lt;p&gt;[ 3213.129550] ZFS: Loaded module v0.6.5-1, ZFS pool version 5000, ZFS filesystem version 5&lt;br/&gt;
[ 3213.395985]  sdb:&lt;br/&gt;
[ 3213.540905]  sdb: sdb1 sdb9&lt;br/&gt;
[ 3213.707262]  sdb: sdb1 sdb9&lt;br/&gt;
[ 3213.844325]  sdc:&lt;br/&gt;
[ 3213.914772]  sdc: sdc1 sdc9&lt;br/&gt;
[ 3214.106819]  sdc: sdc1 sdc9&lt;br/&gt;
[ 3214.251383]  sdd:&lt;br/&gt;
[ 3214.295644]  sdd: sdd1 sdd9&lt;br/&gt;
[ 3214.463067]  sdd: sdd1 sdd9&lt;br/&gt;
[ 3214.600756]  sde:&lt;br/&gt;
[ 3214.653600]  sde: sde1 sde9&lt;br/&gt;
[ 3214.862012]  sde: sde1 sde9&lt;br/&gt;
[ 3215.053045] SPL: using hostid 0x00000000&lt;br/&gt;
[ 3216.449252] Lustre: MGS: Connection restored to ee5079dc-35fc-2fc6-00a3-b0600c8552a3 (at 0@lo)&lt;br/&gt;
[ 3216.844470] Lustre: ctl-lsdraid-MDT0000: No data found on store. Initialize space&lt;br/&gt;
[ 3216.885787] Lustre: lsdraid-MDT0000: new disk, initializing&lt;br/&gt;
[ 3217.429235] Lustre: ctl-lsdraid-MDT0000: super-sequence allocation rc = 0 &lt;span class=&quot;error&quot;&gt;&amp;#91;0x0000000200000400-0x0000000240000400&amp;#93;&lt;/span&gt;:0:mdt&lt;br/&gt;
[ 3242.250332] Lustre: lsdraid-MDT0000: Connection restored to ee5079dc-35fc-2fc6-00a3-b0600c8552a3 (at 0@lo)&lt;br/&gt;
[ 3385.282575] Lustre: MGS: Connection restored to 03723fcc-1218-74ec-5511-35a6a3f56a98 (at 192.168.1.3@o2ib)&lt;/p&gt;

&lt;p&gt;OSS 1&lt;br/&gt;
Restarting network (via systemctl):  Job for network.service failed because the control process exited with error code. See &quot;systemctl status network.service&quot; and &quot;journalctl -xe&quot; for details.&lt;br/&gt;
                                                           &lt;span class=&quot;error&quot;&gt;&amp;#91;FAILED&amp;#93;&lt;/span&gt;&lt;br/&gt;
  pool: ost0&lt;br/&gt;
 state: ONLINE&lt;br/&gt;
  scan: none requested&lt;br/&gt;
config:&lt;/p&gt;

&lt;p&gt;	NAME                      STATE     READ WRITE CKSUM&lt;br/&gt;
	ost0                      ONLINE       0     0     0&lt;br/&gt;
	  draid1-0 &lt;/p&gt;
{any}
&lt;p&gt;          ONLINE       0     0     0&lt;br/&gt;
	    mpathaj               ONLINE       0     0     0&lt;br/&gt;
	    mpathai               ONLINE       0     0     0&lt;br/&gt;
	    mpathah               ONLINE       0     0     0&lt;br/&gt;
	    mpathag               ONLINE       0     0     0&lt;br/&gt;
	    mpathaq               ONLINE       0     0     0&lt;br/&gt;
	    mpathap               ONLINE       0     0     0&lt;br/&gt;
	    mpathak               ONLINE       0     0     0&lt;br/&gt;
	    mpathz                ONLINE       0     0     0&lt;br/&gt;
	    mpatham               ONLINE       0     0     0&lt;br/&gt;
	    mpathal               ONLINE       0     0     0&lt;br/&gt;
	    mpathao               ONLINE       0     0     0&lt;br/&gt;
	  mirror-1 &lt;/p&gt;
{ddt,dmu,mos}
&lt;p&gt;  ONLINE       0     0     0&lt;br/&gt;
	    mpathae               ONLINE       0     0     0&lt;br/&gt;
	    mpathaa               ONLINE       0     0     0&lt;br/&gt;
	spares&lt;br/&gt;
	  $draid1-0-s0            AVAIL   &lt;/p&gt;

&lt;p&gt;errors: No known data errors&lt;br/&gt;
ost0  feature@large_blocks            enabled                         local&lt;/p&gt;

&lt;p&gt;   Permanent disk data:&lt;br/&gt;
Target:     lsdraid-OST0000&lt;br/&gt;
Index:      0&lt;br/&gt;
Lustre FS:  lsdraid&lt;br/&gt;
Mount type: zfs&lt;br/&gt;
Flags:      0x42&lt;br/&gt;
              (OST update )&lt;br/&gt;
Persistent mount opts: &lt;br/&gt;
Parameters: mgsnode=192.168.1.5@o2ib&lt;/p&gt;

&lt;p&gt;mkfs_cmd = zfs create -o canmount=off -o xattr=sa ost0/ost0&lt;br/&gt;
Writing ost0/ost0 properties&lt;br/&gt;
  lustre:version=1&lt;br/&gt;
  lustre:flags=66&lt;br/&gt;
  lustre:index=0&lt;br/&gt;
  lustre:fsname=lsdraid&lt;br/&gt;
  lustre:svname=lsdraid-OST0000&lt;br/&gt;
  lustre:mgsnode=192.168.1.5@o2ib&lt;br/&gt;
mount.lustre: mount ost0/ost0 at /mnt/lustre/ost0 failed: No such file or directory&lt;br/&gt;
Is the MGS specification correct?&lt;br/&gt;
Is the filesystem name correct?&lt;br/&gt;
If upgrading, is the copied client log valid? (see upgrade docs)&lt;/p&gt;

&lt;p&gt;[ 3359.353412] ZFS: Unloaded module v0.6.5-1&lt;br/&gt;
[ 3359.365204] SPL: Unloaded module v0.6.5-1&lt;br/&gt;
[ 3359.383985] SPL: Loaded module v0.6.5-1&lt;br/&gt;
[ 3360.922575] ZFS: Loaded module v0.6.5-1, ZFS pool version 5000, ZFS filesystem version 5&lt;br/&gt;
[ 3362.920084] ZFS: spa metadata allocation class  feature not enabled (ddt,dmu,mos)&lt;br/&gt;
[ 3362.929898] ZFS: vdev_alloc primary &apos;ddt,dmu,mos&apos;&lt;br/&gt;
[ 3363.067121] SPL: using hostid 0x00000000&lt;br/&gt;
[ 3363.778060] ZFS: vdev_construct_zaps assign primary &apos;ddt,dmu,mos&apos;&lt;br/&gt;
[ 3366.736766] LustreError: 13a-8: Failed to get MGS log lsdraid-OST0000 and no local copy.&lt;br/&gt;
[ 3366.746709] LustreError: 15c-8: MGC192.168.1.5@o2ib: The configuration from log &apos;lsdraid-OST0000&apos; failed (-2). This may be the result of communication errors between this node and the MGS, a bad configuration, or other errors. See the syslog for more information.&lt;br/&gt;
[ 3366.775097] LustreError: 47845:0:(obd_mount_server.c:1352:server_start_targets()) failed to start server lsdraid-OST0000: -2&lt;br/&gt;
[ 3366.788532] LustreError: 47845:0:(obd_mount_server.c:1844:server_fill_super()) Unable to start targets: -2&lt;br/&gt;
[ 3366.800271] LustreError: 47845:0:(obd_mount_server.c:1558:server_put_super()) no obd lsdraid-OST0000&lt;br/&gt;
[ 3367.038876] Lustre: server umount lsdraid-OST0000 complete&lt;br/&gt;
[ 3367.046063] LustreError: 47845:0:(obd_mount.c:1453:lustre_fill_super()) Unable to mount  (-2)&lt;/p&gt;</comment>
                            <comment id="162194" author="kit.westneat" created="Wed, 17 Aug 2016 15:48:52 +0000"  >&lt;p&gt;Can you post the MDS and OSS logs from that time period?&lt;/p&gt;</comment>
                            <comment id="162199" author="jsalians_intel" created="Wed, 17 Aug 2016 15:56:04 +0000"  >&lt;p&gt;Please see: &lt;br/&gt;
11192890 Aug 17 10:50 oss_lctl_log.afterpatch&lt;br/&gt;
 8799197 Aug 17 10:54 mds_lctl_log_afterpatch &lt;/p&gt;

&lt;p&gt;let me know if you need anything else&lt;/p&gt;</comment>
                            <comment id="162201" author="kit.westneat" created="Wed, 17 Aug 2016 16:03:48 +0000"  >&lt;p&gt;It looks like the nodemap config loaded properly, but there was an error in registration:&lt;br/&gt;
On MDS:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;20000000:02020000:21.0:1471449182.711299:0:40275:0:(mgs_handler.c:297:mgs_check_target()) 13b-9: lsdraid-OST0000 claims to have registered, but &lt;span class=&quot;code-keyword&quot;&gt;this&lt;/span&gt; MGS does not know about it, preventing registration.
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;On OSS:&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;10000000:01000000:37.0:1471448892.078388:0:48119:0:(mgc_request.c:2086:mgc_process_log()) MGC192.168.1.5@o2ib: configuration from log &lt;span class=&quot;code-quote&quot;&gt;&apos;nodemap&apos;&lt;/span&gt; succeeded (0).
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;Is it possible there&apos;s some kind of timing issue between reformatting the OSSes and the MDS?&lt;/p&gt;</comment>
                            <comment id="162266" author="jsalians_intel" created="Wed, 17 Aug 2016 20:03:31 +0000"  >&lt;p&gt;I would be surprised if there could be a timing issue &amp;#8211; I am running scripts by hand first on mds and once it is done oss1 and I try it several times by hand to capture debug lugs.  This process has worked for months and months until nodemap came along.  However, that doesn&apos;t mean there isn&apos;t another issue here.  &lt;/p&gt;</comment>
                            <comment id="162555" author="jsalians_intel" created="Fri, 19 Aug 2016 18:02:48 +0000"  >&lt;p&gt;Today I re-built with tip of master + this patch and was able to get an OSS to mount successfully &amp;#8211; using the same scripts: &lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;./setup_oss1.sh&lt;br/&gt;
Restarting network (via systemctl):  Job for network.service failed because the control process exited with error code. See &quot;systemctl status network.service&quot; and &quot;journalctl -xe&quot; for details.&lt;br/&gt;
                                                           &lt;span class=&quot;error&quot;&gt;&amp;#91;FAILED&amp;#93;&lt;/span&gt;&lt;br/&gt;
modprobe: FATAL: Module lnet is in use.&lt;br/&gt;
modprobe: FATAL: Module zfs is in use.&lt;br/&gt;
  pool: ost0&lt;br/&gt;
 state: ONLINE&lt;br/&gt;
  scan: none requested&lt;br/&gt;
config:&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;	NAME                      STATE     READ WRITE CKSUM&lt;br/&gt;
	ost0                      ONLINE       0     0     0&lt;br/&gt;
	  draid1-0 &lt;/p&gt;
{any}
&lt;p&gt;          ONLINE       0     0     0&lt;br/&gt;
	    mpathaj               ONLINE       0     0     0&lt;br/&gt;
	    mpathai               ONLINE       0     0     0&lt;br/&gt;
	    mpathah               ONLINE       0     0     0&lt;br/&gt;
	    mpathag               ONLINE       0     0     0&lt;br/&gt;
	    mpathaq               ONLINE       0     0     0&lt;br/&gt;
	    mpathap               ONLINE       0     0     0&lt;br/&gt;
	    mpathak               ONLINE       0     0     0&lt;br/&gt;
	    mpathz                ONLINE       0     0     0&lt;br/&gt;
	    mpatham               ONLINE       0     0     0&lt;br/&gt;
	    mpathal               ONLINE       0     0     0&lt;br/&gt;
	    mpathao               ONLINE       0     0     0&lt;br/&gt;
	  mirror-1 &lt;/p&gt;
{ddt,dmu,mos}
&lt;p&gt;  ONLINE       0     0     0&lt;br/&gt;
	    mpathae               ONLINE       0     0     0&lt;br/&gt;
	    mpathaa               ONLINE       0     0     0&lt;br/&gt;
	spares&lt;br/&gt;
	  $draid1-0-s0            AVAIL   &lt;/p&gt;

&lt;p&gt;errors: No known data errors&lt;br/&gt;
ost0  feature@large_blocks            enabled                         local&lt;/p&gt;

&lt;p&gt;   Permanent disk data:&lt;br/&gt;
Target:     lsdraid-OST0000&lt;br/&gt;
Index:      0&lt;br/&gt;
Lustre FS:  lsdraid&lt;br/&gt;
Mount type: zfs&lt;br/&gt;
Flags:      0x42&lt;br/&gt;
              (OST update )&lt;br/&gt;
Persistent mount opts: &lt;br/&gt;
Parameters: mgsnode=192.168.1.5@o2ib&lt;/p&gt;

&lt;p&gt;mkfs_cmd = zfs create -o canmount=off -o xattr=sa ost0/ost0&lt;br/&gt;
Writing ost0/ost0 properties&lt;br/&gt;
  lustre:version=1&lt;br/&gt;
  lustre:flags=66&lt;br/&gt;
  lustre:index=0&lt;br/&gt;
  lustre:fsname=lsdraid&lt;br/&gt;
  lustre:svname=lsdraid-OST0000&lt;br/&gt;
  lustre:mgsnode=192.168.1.5@o2ib&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;mount |grep ost0&lt;br/&gt;
ost0 on /ost0 type zfs (rw,xattr,noacl)&lt;br/&gt;
ost0/ost0 on /mnt/lustre/ost0 type lustre (ro) &lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Will install the rest of the system and if we can get all of the nodes up then will mark this as fixed &amp;#8211; but we will need this fix merged.&lt;/p&gt;</comment>
                            <comment id="162589" author="jsalians_intel" created="Fri, 19 Aug 2016 22:06:44 +0000"  >&lt;p&gt;The good news is one OST can now connect to MDS.  I saw one odd messages when clients are mounting I see this: &lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;26059.015447&amp;#93;&lt;/span&gt; libcfs: module verification failed: signature and/or required key missing - tainting kernel&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26059.030063&amp;#93;&lt;/span&gt; LNet: HW CPU cores: 72, npartitions: 8&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26059.038309&amp;#93;&lt;/span&gt; alg: No test for adler32 (adler32-zlib)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26059.044985&amp;#93;&lt;/span&gt; alg: No test for crc32 (crc32-table)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26068.348811&amp;#93;&lt;/span&gt; LNet: HW CPU cores: 72, npartitions: 8&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26068.357409&amp;#93;&lt;/span&gt; alg: No test for adler32 (adler32-zlib)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26068.364132&amp;#93;&lt;/span&gt; alg: No test for crc32 (crc32-table)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26076.388483&amp;#93;&lt;/span&gt; Lustre: Lustre: Build Version: 2.8.56&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26076.471053&amp;#93;&lt;/span&gt; LNet: Added LNI 192.168.1.6@o2ib &lt;span class=&quot;error&quot;&gt;&amp;#91;8/512/0/180&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26076.536301&amp;#93;&lt;/span&gt; LustreError: 44972:0:(mgc_request.c:1533:mgc_apply_recover_logs()) mgc: cannot find uuid by nid 192.168.1.4@o2ib&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26076.550097&amp;#93;&lt;/span&gt; Lustre: 44972:0:(mgc_request.c:1760:mgc_process_recover_nodemap_log()) MGC192.168.1.5@o2ib: error processing recovery log lsdraid-cliir: rc = -2 &lt;/p&gt;

&lt;p&gt;.4 is OSS2 and .5 is MDS node&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-7 ~&amp;#93;&lt;/span&gt;# df -h /mnt/lustre/&lt;br/&gt;
Filesystem                 Size  Used Avail Use% Mounted on&lt;br/&gt;
192.168.1.5@o2ib:/lsdraid  7.0T   16M  7.0T   1% /mnt/lustre &lt;/p&gt;

&lt;p&gt;so we are missing the zpool from OSS2: &lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 ~&amp;#93;&lt;/span&gt;# zpool list&lt;br/&gt;
NAME   SIZE  ALLOC   FREE  EXPANDSZ   FRAG    CAP  DEDUP  HEALTH  ALTROOT&lt;br/&gt;
ost0  9.97T  21.0M  9.97T         -     0%     0%  1.00x  ONLINE  -&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 ~&amp;#93;&lt;/span&gt;# zfs list &lt;br/&gt;
NAME        USED  AVAIL  REFER  MOUNTPOINT&lt;br/&gt;
ost0       17.8M  7.01T    19K  /ost0&lt;br/&gt;
ost0/ost0  17.6M  7.01T  17.6M  /ost0/ost0&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 ~&amp;#93;&lt;/span&gt;# mount |grep lustre&lt;br/&gt;
ost0/ost0 on /mnt/lustre/ost0 type lustre (ro)&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zpool list &lt;br/&gt;
NAME   SIZE  ALLOC   FREE  EXPANDSZ   FRAG    CAP  DEDUP  HEALTH  ALTROOT&lt;br/&gt;
ost1  9.97T   846K  9.97T         -     0%     0%  1.00x  ONLINE  -&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zfs list &lt;br/&gt;
NAME        USED  AVAIL  REFER  MOUNTPOINT&lt;br/&gt;
ost1       1.51M  7.01T    19K  /ost1&lt;br/&gt;
ost1/ost1  1.37M  7.01T  1.37M  /ost1/ost1&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# mount |grep lustre&lt;br/&gt;
ost1/ost1 on /mnt/lustre/ost1 type lustre (ro)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25673.466998&amp;#93;&lt;/span&gt; ZFS: spa metadata allocation class  feature not enabled (ddt,dmu,mos)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25673.477813&amp;#93;&lt;/span&gt; ZFS: vdev_alloc primary &apos;ddt,dmu,mos&apos;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25674.379121&amp;#93;&lt;/span&gt; ZFS: vdev_construct_zaps assign primary &apos;ddt,dmu,mos&apos;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25677.322609&amp;#93;&lt;/span&gt; LustreError: 52126:0:(mgc_request.c:257:do_config_log_add()) MGC192.168.1.5@o2ib: failed processing log, type 4: rc = -22&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25677.342892&amp;#93;&lt;/span&gt; Lustre: lsdraid-OST0000: new disk, initializing&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25677.351849&amp;#93;&lt;/span&gt; Lustre: srv-lsdraid-OST0000: No data found on store. Initialize space&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25677.450718&amp;#93;&lt;/span&gt; Lustre: lsdraid-OST0000: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-450&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.093757&amp;#93;&lt;/span&gt; LustreError: 11-0: lsdraid-MDT0000-lwp-OST0000: operation obd_ping to node 192.168.1.5@o2ib failed: rc = -107&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.108646&amp;#93;&lt;/span&gt; LustreError: Skipped 21 previous similar messages&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.117392&amp;#93;&lt;/span&gt; Lustre: lsdraid-MDT0000-lwp-OST0000: Connection to lsdraid-MDT0000 (at 192.168.1.5@o2ib) was lost; in progress operations using this service will wait for recovery to complete&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.140650&amp;#93;&lt;/span&gt; Lustre: Skipped 21 previous similar messages&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.149666&amp;#93;&lt;/span&gt; LustreError: 167-0: lsdraid-MDT0000-lwp-OST0000: This client was evicted by lsdraid-MDT0000; in progress operations using this service will fail.&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.170128&amp;#93;&lt;/span&gt; LustreError: Skipped 21 previous similar messages&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.180193&amp;#93;&lt;/span&gt; Lustre: lsdraid-MDT0000-lwp-OST0000: Connection restored to 192.168.1.5@o2ib (at 192.168.1.5@o2ib)&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;26177.194074&amp;#93;&lt;/span&gt; Lustre: Skipped 21 previous similar messages&lt;/p&gt;</comment>
                            <comment id="162606" author="jsalians_intel" created="Sat, 20 Aug 2016 20:00:15 +0000"  >&lt;p&gt;It does indeed appear that I can only connect 1 ost &amp;#8211; when the second ost tries to connect I see this: &lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zpool create -f -o cachefile=none -O recordsize=16MB ost1 draid1 /dev/mapper/mpathl /dev/mapper/mpathk /dev/mapper/mpathj /dev/mapper/mpathi /dev/mapper/mpathr /dev/mapper/mpathm /dev/mapper/mpathb /dev/mapper/mpatha /dev/mapper/mpatho /dev/mapper/mpathn /dev/mapper/mpathq spare &apos;$draid1-0-s0&apos; metadata mirror /dev/mapper/mpathg /dev/mapper/mpathx&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# zpool feature@large_blocks=enabled ost1&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# mkfs.lustre --backfstype=zfs  --reformat --fsname=lsdraid --ost --index=1 --mgsnode=192.168.1.5@o2ib ost1/ost1&lt;/p&gt;

&lt;p&gt;   Permanent disk data:&lt;br/&gt;
Target:     lsdraid:OST0001&lt;br/&gt;
Index:      1&lt;br/&gt;
Lustre FS:  lsdraid&lt;br/&gt;
Mount type: zfs&lt;br/&gt;
Flags:      0x62&lt;br/&gt;
              (OST first_time update )&lt;br/&gt;
Persistent mount opts: &lt;br/&gt;
Parameters: mgsnode=192.168.1.5@o2ib&lt;/p&gt;

&lt;p&gt;mkfs_cmd = zfs create -o canmount=off -o xattr=sa ost1/ost1&lt;br/&gt;
Writing ost1/ost1 properties&lt;br/&gt;
  lustre:version=1&lt;br/&gt;
  lustre:flags=98&lt;br/&gt;
  lustre:index=1&lt;br/&gt;
  lustre:fsname=lsdraid&lt;br/&gt;
  lustre:svname=lsdraid:OST0001&lt;br/&gt;
  lustre:mgsnode=192.168.1.5@o2ib&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# mount -t lustre ost1/ost1 /mnt/lustre/ost1&lt;/p&gt;

&lt;p&gt;Aug 20 19:55:13 wolf-4 kernel: ZFS: spa metadata allocation class  feature not enabled (ddt,dmu,mos)&lt;br/&gt;
Aug 20 19:55:13 wolf-4 kernel: ZFS: vdev_alloc primary &apos;ddt,dmu,mos&apos;&lt;br/&gt;
Aug 20 19:55:14 wolf-4 kernel: ZFS: vdev_construct_zaps assign primary &apos;ddt,dmu,mos&apos;&lt;br/&gt;
Aug 20 19:55:27 wolf-4 ntpd_intres&lt;span class=&quot;error&quot;&gt;&amp;#91;2643&amp;#93;&lt;/span&gt;: ntp_intres.request: permission denied&lt;br/&gt;
Aug 20 19:55:27 wolf-4 ntpd_intres&lt;span class=&quot;error&quot;&gt;&amp;#91;2643&amp;#93;&lt;/span&gt;: ntp_intres.request: permission denied&lt;br/&gt;
Aug 20 19:55:41 wolf-4 kernel: LustreError: 41114:0:(mgc_request.c:257:do_config_log_add()) MGC192.168.1.5@o2ib: failed processing log, type 4: rc = -22&lt;br/&gt;
Aug 20 19:55:41 wolf-4 kernel: Lustre: lsdraid-OST0001: new disk, initializing&lt;br/&gt;
Aug 20 19:55:41 wolf-4 kernel: Lustre: srv-lsdraid-OST0001: No data found on store. Initialize space&lt;br/&gt;
Aug 20 19:55:46 wolf-4 kernel: Lustre: lsdraid-OST0001: Connection restored to 192.168.1.5@o2ib (at 192.168.1.5@o2ib)&lt;br/&gt;
Aug 20 19:55:46 wolf-4 kernel: Lustre: Skipped 2 previous similar messages&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# mount |grep lustre&lt;br/&gt;
ost1/ost1 on /mnt/lustre/ost1 type lustre (ro)&lt;/p&gt;

&lt;p&gt;from mds:&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-5 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# lctl dl &lt;br/&gt;
  0 UP osd-zfs lsdraid-MDT0000-osd lsdraid-MDT0000-osd_UUID 10&lt;br/&gt;
  1 UP mgs MGS MGS 9&lt;br/&gt;
  2 UP mgc MGC192.168.1.5@o2ib ac271bb5-f2ee-996f-94cf-6c9cd20a078a 5&lt;br/&gt;
  3 UP mds MDS MDS_uuid 3&lt;br/&gt;
  4 UP lod lsdraid-MDT0000-mdtlov lsdraid-MDT0000-mdtlov_UUID 4&lt;br/&gt;
  5 UP mdt lsdraid-MDT0000 lsdraid-MDT0000_UUID 9&lt;br/&gt;
  6 UP mdd lsdraid-MDD0000 lsdraid-MDD0000_UUID 4&lt;br/&gt;
  7 UP qmt lsdraid-QMT0000 lsdraid-QMT0000_UUID 4&lt;br/&gt;
  8 UP lwp lsdraid-MDT0000-lwp-MDT0000 lsdraid-MDT0000-lwp-MDT0000_UUID 5&lt;br/&gt;
  9 UP osp lsdraid-OST0000-osc-MDT0000 lsdraid-MDT0000-mdtlov_UUID 5&lt;br/&gt;
 10 UP osp lsdraid-OST0001-osc-MDT0000 lsdraid-MDT0000-mdtlov_UUID 5&lt;/p&gt;

&lt;p&gt;from oss1&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# lctl dl &lt;br/&gt;
  0 UP osd-zfs lsdraid-OST0000-osd lsdraid-OST0000-osd_UUID 5&lt;br/&gt;
  1 UP mgc MGC192.168.1.5@o2ib c2ddb1b6-eda7-06a1-c4c7-7a10c79169f2 5&lt;br/&gt;
  2 UP ost OSS OSS_uuid 3&lt;br/&gt;
  3 UP obdfilter lsdraid-OST0000 lsdraid-OST0000_UUID 5&lt;br/&gt;
  4 UP lwp lsdraid-MDT0000-lwp-OST0000 lsdraid-MDT0000-lwp-OST0000_UUID 5&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-3 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# &lt;/p&gt;


&lt;p&gt;from oss2&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@wolf-4 mdisolation_step2_draid_testing&amp;#93;&lt;/span&gt;# lctl dl &lt;br/&gt;
  0 UP osd-zfs lsdraid-OST0001-osd lsdraid-OST0001-osd_UUID 5&lt;br/&gt;
  1 UP mgc MGC192.168.1.5@o2ib 9baa6009-12fa-026e-a230-d8562d7b63c7 5&lt;br/&gt;
  2 UP ost OSS OSS_uuid 3&lt;br/&gt;
  3 UP obdfilter lsdraid-OST0001 lsdraid-OST0001_UUID 5&lt;br/&gt;
  4 UP lwp lsdraid-MDT0000-lwp-OST0001 lsdraid-MDT0000-lwp-OST0001_UUID 5 &lt;/p&gt;
</comment>
                            <comment id="162607" author="kit.westneat" created="Sat, 20 Aug 2016 23:32:48 +0000"  >&lt;p&gt;Hey John,&lt;/p&gt;

&lt;p&gt;I see that there&apos;s a nodemap error, which is definitely a problem, but it looks like the OST still connects ok:&lt;br/&gt;
Aug 20 19:55:46 wolf-4 kernel: Lustre: lsdraid-OST0001: Connection restored to 192.168.1.5@o2ib (at 192.168.1.5@o2ib)&lt;/p&gt;

&lt;p&gt;Was there a problem on the clients as well? Do you happen to have the debug logs for this time period for the OSS and MDS?&lt;/p&gt;

&lt;p&gt;BTW there&apos;s another nodemap related bug / patch here:&lt;br/&gt;
&lt;a href=&quot;http://review.whamcloud.com/#/c/22004/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/#/c/22004/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;I don&apos;t think it&apos;s related to what you&apos;re seeing, but it could be worth adding to the mix. &lt;/p&gt;</comment>
                            <comment id="163325" author="jsalians_intel" created="Fri, 26 Aug 2016 23:44:32 +0000"  >&lt;p&gt;It appears sometimes the ost still does connect &amp;#8211; I just tried now and of course it connected because I was trying to collect more information.  I will lower priority for now. &lt;/p&gt;</comment>
                            <comment id="165227" author="gerrit" created="Thu, 8 Sep 2016 02:05:52 +0000"  >&lt;p&gt;Oleg Drokin (oleg.drokin@intel.com) merged in patch &lt;a href=&quot;http://review.whamcloud.com/21939/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/21939/&lt;/a&gt;&lt;br/&gt;
Subject: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-8498&quot; title=&quot;configuration from log &amp;#39;nodemap&amp;#39; failed (-22)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-8498&quot;&gt;&lt;del&gt;LU-8498&lt;/del&gt;&lt;/a&gt; nodemap: new zfs index files not properly initialized&lt;br/&gt;
Project: fs/lustre-release&lt;br/&gt;
Branch: master&lt;br/&gt;
Current Patch Set: &lt;br/&gt;
Commit: dd69c1da4ab22c66801df6a6e316d210ea71454d&lt;/p&gt;</comment>
                            <comment id="165253" author="pjones" created="Thu, 8 Sep 2016 04:19:20 +0000"  >&lt;p&gt;Landed for 2.9&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="18740">LU-3291</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="22558" name="log.Jinshan" size="224" author="jsalians_intel" created="Thu, 11 Aug 2016 19:08:17 +0000"/>
                            <attachment id="22562" name="log.mds.LU-8498" size="232" author="jsalians_intel" created="Fri, 12 Aug 2016 21:51:57 +0000"/>
                            <attachment id="22565" name="log_failedmount_vanilla_master" size="247" author="jsalians_intel" created="Mon, 15 Aug 2016 18:53:08 +0000"/>
                            <attachment id="22601" name="mds_lctl_log_afterpatch" size="232" author="jsalians_intel" created="Wed, 17 Aug 2016 15:55:05 +0000"/>
                            <attachment id="22600" name="oss_lctl_log.afterpatch" size="248" author="jsalians_intel" created="Wed, 17 Aug 2016 15:51:15 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                            <customfield id="customfield_10890" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10390" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hzykin:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10090" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10060" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Severity</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[1]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        </customfields>
    </item>
</channel>
</rss>