<!-- 
RSS generated by JIRA (9.4.14#940014-sha1:734e6822bbf0d45eff9af51f82432957f73aa32c) at Sat Feb 10 03:28:51 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>Whamcloud Community JIRA</title>
    <link>https://jira.whamcloud.com</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.4.14</version>
        <build-number>940014</build-number>
        <build-date>05-12-2023</build-date>
    </build-info>


<item>
            <title>[LU-16652] sanity-lnet test_253: Expect 1 dropped GET but found 2</title>
                <link>https://jira.whamcloud.com/browse/LU-16652</link>
                <project id="10000" key="LU">Lustre</project>
                    <description>&lt;p&gt;This issue was created by maloo for Andreas Dilger &amp;lt;adilger@whamcloud.com&amp;gt;&lt;/p&gt;

&lt;p&gt;This issue relates to the following test suite run:&lt;br/&gt;
&lt;a href=&quot;https://testing.whamcloud.com/test_sets/48cb196d-84d7-47a6-b3b8-da7376ea54c4&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://testing.whamcloud.com/test_sets/48cb196d-84d7-47a6-b3b8-da7376ea54c4&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;test_253 failed intermittently with the following error:&lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;Expect 1 dropped GET but found 2
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;

&lt;p&gt;Test session details:&lt;br/&gt;
clients: &lt;a href=&quot;https://build.whamcloud.com/job/lustre-reviews/92208&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://build.whamcloud.com/job/lustre-reviews/92208&lt;/a&gt; - 5.15.0-52-generic&lt;br/&gt;
servers: &lt;a href=&quot;https://build.whamcloud.com/job/lustre-reviews/92208&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://build.whamcloud.com/job/lustre-reviews/92208&lt;/a&gt; - 4.18.0-348.23.1.el8_lustre.x86_64&lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;CMD: trevis-58vm2 /usr/sbin/lctl list_nids
/usr/sbin/lnetctl discover 10.240.40.249@tcp
discover:
    - primary nid: 10.240.40.249@tcp
      Multi-Rail: True
      peer ni:
        - nid: 10.240.40.249@tcp
net:
    - net type: lo
      local NI(s):
        - nid: 0@lo
          status: up
    - net type: tcp
      local NI(s):
        - nid: 10.240.40.248@tcp
          status: up
          interfaces:
              0: eth0
    - primary nid: 10.240.40.249@tcp
        - nid: 10.240.40.249@tcp
          health stats:
              health value: 1000
debug=+net
/usr/sbin/lnetctl set transaction_timeout 10
Added delay rule 10.240.40.248@tcp-&amp;gt;10.240.40.249@tcp (1/1)
Found 8 peer_credits for 10.240.40.249@tcp
Issued 8 pings to 10.240.40.249@tcp
manage:
    - ping:
          errno: -1
          descr: failed to ping 10.240.40.249@tcp: Connection timed out
                 
manage:
    - ping:
          errno: -1
          descr: failed to ping 10.240.40.249@tcp: Connection timed out
                 
Removed 1 delay rules
ping:
    - primary nid: 10.240.40.249@tcp
      Multi-Rail: True
      peer ni:
        - nid: 10.240.40.249@tcp
 sanity-lnet test_253: @@@@@@ FAIL: Expect 1 dropped GET but found 2 
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;







&lt;p&gt;VVVVVVV DO NOT REMOVE LINES BELOW, Added by Maloo for auto-association VVVVVVV&lt;br/&gt;
sanity-lnet test_253 - Expect 1 dropped GET but found 2&lt;/p&gt;</description>
                <environment></environment>
        <key id="75137">LU-16652</key>
            <summary>sanity-lnet test_253: Expect 1 dropped GET but found 2</summary>
                <type id="1" iconUrl="https://jira.whamcloud.com/secure/viewavatar?size=xsmall&amp;avatarId=11303&amp;avatarType=issuetype">Bug</type>
                                            <priority id="4" iconUrl="https://jira.whamcloud.com/images/icons/priorities/minor.svg">Minor</priority>
                        <status id="5" iconUrl="https://jira.whamcloud.com/images/icons/statuses/resolved.png" description="A resolution has been taken, and it is awaiting verification by reporter. From here issues are either reopened, or are closed.">Resolved</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="1">Fixed</resolution>
                                        <assignee username="hornc">Chris Horn</assignee>
                                    <reporter username="maloo">Maloo</reporter>
                        <labels>
                    </labels>
                <created>Tue, 21 Mar 2023 17:29:35 +0000</created>
                <updated>Tue, 11 Apr 2023 21:58:42 +0000</updated>
                            <resolved>Tue, 11 Apr 2023 21:58:42 +0000</resolved>
                                                    <fixVersion>Lustre 2.16.0</fixVersion>
                                        <due></due>
                            <votes>0</votes>
                                    <watches>7</watches>
                                                                            <comments>
                            <comment id="366707" author="adilger" created="Tue, 21 Mar 2023 17:32:48 +0000"  >&lt;p&gt;Patches landed to master on 2023-02-08 when this subtest first started failing:&lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;$ git log --oneline --after 2023-02-06 --before 2023-02-09
0568f4ca25 LU-16500 utils: set default ost index for lfs migrate
9ce04000fb LU-930 ptlrpc: clarify AT error message
e6b6b7ee25 LU-16367 utils: clean up ldiskfs feature handling
a05d02ea0e LU-16221 kernel: new kernel [RHEL 9.1 5.14.0-162.12.1.el9_1]
919b93b951 LU-16510 build: fortified memcpy from linux 6.1
738e69d4b9 LU-16292 llite: delete_from_page_cache not exported
b13a5b351e LU-16188 mdt: fix incompatible HSM request handling
a3a51806ef LU-16118 build: Workaround __write_overflow_field errors
391392e117 LU-16354 ldiskfs: RHEL9.1 server support
54eb6da1f8 LU-16477 ldiskfs: Add ext4-enc-flag patch for RHEL9
c10c6eeb37 LU-15728 llite: fix relatime support
0c05dc21ab LU-6142 ldlm: minor list_entry improvements in ldlm_request.c
685fb4b17f LU-6142 ldlm: use list_for_each_entry in ldlm_lock.c
93230059ab LU-12275 tests: skip new nodemap params on old MGS
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                            <comment id="366710" author="adilger" created="Tue, 21 Mar 2023 17:42:37 +0000"  >&lt;p&gt;It looks like this is &lt;b&gt;only&lt;/b&gt; failing on review-ldiskfs-ubuntu with Ubuntu22.04 clients, about 1/10 of sanity-lnet tests on that distro.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://testing.whamcloud.com/search?client_branch_type_id=24a6947e-04a9-11e1-bb5f-52540025f9af&amp;amp;client_distribution_type_id=546c7bd8-1a0d-46be-afb2-f9e842762553&amp;amp;server_branch_type_id=24a6947e-04a9-11e1-bb5f-52540025f9af&amp;amp;test_set_script_id=a2b1c4b2-b449-11e9-b88c-52540065bddc&amp;amp;sub_test_script_id=5a631bd3-f3e2-48aa-937f-fa3662bd2d47&amp;amp;start_date=2023-03-14&amp;amp;end_date=2023-03-21&amp;amp;source=sub_tests#redirect&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://testing.whamcloud.com/search?client_branch_type_id=24a6947e-04a9-11e1-bb5f-52540025f9af&amp;amp;client_distribution_type_id=546c7bd8-1a0d-46be-afb2-f9e842762553&amp;amp;server_branch_type_id=24a6947e-04a9-11e1-bb5f-52540025f9af&amp;amp;test_set_script_id=a2b1c4b2-b449-11e9-b88c-52540065bddc&amp;amp;sub_test_script_id=5a631bd3-f3e2-48aa-937f-fa3662bd2d47&amp;amp;start_date=2023-03-14&amp;amp;end_date=2023-03-21&amp;amp;source=sub_tests#redirect&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="367792" author="adilger" created="Wed, 29 Mar 2023 20:59:50 +0000"  >&lt;p&gt;&lt;del&gt;Chris, presumably this is the same cause as &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-16551&quot; title=&quot;sanity-lnet test_254: Expect 1 dropped GET but found 2&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-16551&quot;&gt;&lt;del&gt;LU-16551&lt;/del&gt;&lt;/a&gt; and could be closed as a duplicate now that your patch landed for that ticket already?&lt;/del&gt;&lt;/p&gt;

&lt;p&gt;Hmm, checking the date, the &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-16551&quot; title=&quot;sanity-lnet test_254: Expect 1 dropped GET but found 2&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-16551&quot;&gt;&lt;del&gt;LU-16551&lt;/del&gt;&lt;/a&gt; patch was landed on 2023-02-22, but these failures continue to be hit on master.  test_253 failed 80/474 runs on Ubuntu22.04 clients since 2023-02-08, 36/242 runs since 2023-03-01, after that patch landed.&lt;/p&gt;

&lt;p&gt;Note the &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-16551&quot; title=&quot;sanity-lnet test_254: Expect 1 dropped GET but found 2&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-16551&quot;&gt;&lt;del&gt;LU-16551&lt;/del&gt;&lt;/a&gt; ticket was for sanity-lnet test_254, while this ticket is for test_253, but they both reported the same error message.  I only see two such test_254 failures in the logs since the start of the year, so it could be that ticket was mis-attributing the source of the failures?  In any case, it looks like the patch did not fix the issue, or there is a similar issue only being hit on Ubuntu.&lt;/p&gt;</comment>
                            <comment id="367797" author="hornc" created="Wed, 29 Mar 2023 21:33:19 +0000"  >&lt;p&gt;The issue is we&apos;re racing with discovery. After the delay rule is added we receive a discovery PUSH (incoming PUT) and the ACK is delayed. This consumes a peer credit, so we end up with two GETs on the peer NI tx queue. These are then dropped after the delay rule is removed and thus we get an extra, unexpected dropped GET.&lt;/p&gt;

&lt;p&gt;We can resolve this by modifying the delay rule to only apply to GET messages.&lt;/p&gt;</comment>
                            <comment id="367799" author="gerrit" created="Wed, 29 Mar 2023 21:43:59 +0000"  >&lt;p&gt;&quot;Chris Horn &amp;lt;chris.horn@hpe.com&amp;gt;&quot; uploaded a new patch: &lt;a href=&quot;https://review.whamcloud.com/c/fs/lustre-release/+/50469&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://review.whamcloud.com/c/fs/lustre-release/+/50469&lt;/a&gt;&lt;br/&gt;
Subject: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-16652&quot; title=&quot;sanity-lnet test_253: Expect 1 dropped GET but found 2&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-16652&quot;&gt;&lt;del&gt;LU-16652&lt;/del&gt;&lt;/a&gt; tests: test 253/254 race with discovery&lt;br/&gt;
Project: fs/lustre-release&lt;br/&gt;
Branch: master&lt;br/&gt;
Current Patch Set: 1&lt;br/&gt;
Commit: d5cda7d1090c433852b3eacdd128f143ef0c8cf5&lt;/p&gt;</comment>
                            <comment id="369143" author="gerrit" created="Tue, 11 Apr 2023 20:11:52 +0000"  >&lt;p&gt;&quot;Oleg Drokin &amp;lt;green@whamcloud.com&amp;gt;&quot; merged in patch &lt;a href=&quot;https://review.whamcloud.com/c/fs/lustre-release/+/50469/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://review.whamcloud.com/c/fs/lustre-release/+/50469/&lt;/a&gt;&lt;br/&gt;
Subject: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-16652&quot; title=&quot;sanity-lnet test_253: Expect 1 dropped GET but found 2&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-16652&quot;&gt;&lt;del&gt;LU-16652&lt;/del&gt;&lt;/a&gt; tests: sanity-lnet test 253/254 race with discovery&lt;br/&gt;
Project: fs/lustre-release&lt;br/&gt;
Branch: master&lt;br/&gt;
Current Patch Set: &lt;br/&gt;
Commit: 050ce41d7359ebf4e10c5e592b5454b133ed3e64&lt;/p&gt;</comment>
                            <comment id="369172" author="pjones" created="Tue, 11 Apr 2023 21:58:42 +0000"  >&lt;p&gt;Landed for 2.16&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Related</name>
                                            <outwardlinks description="is related to ">
                                        <issuelink>
            <issuekey id="74616">LU-16551</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                            <customfield id="customfield_10890" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10390" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i03grz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10090" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10060" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Severity</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10022"><![CDATA[3]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        </customfields>
    </item>
</channel>
</rss>