<!-- 
RSS generated by JIRA (9.4.14#940014-sha1:734e6822bbf0d45eff9af51f82432957f73aa32c) at Sat Feb 10 02:06:43 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>Whamcloud Community JIRA</title>
    <link>https://jira.whamcloud.com</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.4.14</version>
        <build-number>940014</build-number>
        <build-date>05-12-2023</build-date>
    </build-info>


<item>
            <title>[LU-7184] (lod_dev.c:1493:lod_device_free()) ASSERTION( atomic_read(&amp;lu-&gt;ld_ref) == 0 ) failed: lu is ffff88010cf8a000</title>
                <link>https://jira.whamcloud.com/browse/LU-7184</link>
                <project id="10000" key="LU">Lustre</project>
                    <description>&lt;p&gt;Setting the security flavor, similar to below causes an LBUG when the MDT is mounted again:&lt;/p&gt;

&lt;p&gt;/usr/lib64/lustre/tests/llmount.sh&lt;br/&gt;
lctl conf_param lustre.srpc.flavor.default=skpi&lt;br/&gt;
umount -a -f -t lustre&lt;br/&gt;
mount -o loop -t lustre /tmp/lustre-mdt1 /mnt/mds1&lt;/p&gt;

&lt;p&gt;&amp;lt;4&amp;gt;Lustre: server umount lustre-MDT0000 complete&lt;br/&gt;
&amp;lt;6&amp;gt;LDISKFS-fs (loop0): mounted filesystem with ordered data mode. quota=on. Opts: &lt;br/&gt;
&amp;lt;4&amp;gt;Lustre: 4563:0:(llog_cat.c:620:llog_cat_process_or_fork()) catlog 0x2:1 crosses index zero&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(gss_keyring.c:805:gss_sec_lookup_ctx_kr()) failed request key: -126&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(gss_keyring.c:805:gss_sec_lookup_ctx_kr()) Skipped 1 previous similar message&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(sec.c:444:sptlrpc_req_get_ctx()) req ffff88011a183cc0: fail to get context&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(sec.c:444:sptlrpc_req_get_ctx()) Skipped 1 previous similar message&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(osp_dev.c:1437:osp_obd_connect()) lustre-OST0000-osc-MDT0000: can&apos;t connect obd: rc = -111&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(lod_lov.c:293:lod_add_device()) lustre-OST0000-osc-MDT0000: cannot connect to next dev lustre-OST0000_UUID (-111)&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4560:0:(obd_config.c:1624:class_config_llog_handler()) MGC192.168.1.107@tcp: cfg command failed: rc = -111&lt;br/&gt;
&amp;lt;4&amp;gt;Lustre:    cmd=cf00d 0:lustre-MDT0000-mdtlov  1:lustre-OST0000_UUID  2:0  3:1  &lt;br/&gt;
&amp;lt;4&amp;gt;&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 15c-8: MGC192.168.1.107@tcp: The configuration from log &apos;lustre-MDT0000&apos; failed (-111). This may be the result of communication errors between this node and the MGS, a bad configuration, or other errors. See the syslog for more information.&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4513:0:(obd_mount_server.c:1306:server_start_targets()) failed to start server lustre-MDT0000: -111&lt;br/&gt;
&amp;lt;3&amp;gt;LustreError: 4513:0:(obd_mount_server.c:1790:server_fill_super()) Unable to start targets: -111&lt;br/&gt;
&amp;lt;4&amp;gt;Lustre: Failing over lustre-MDT0000&lt;br/&gt;
&amp;lt;0&amp;gt;LustreError: 2659:0:(lod_dev.c:1493:lod_device_free()) ASSERTION( atomic_read(&amp;amp;lu-&amp;gt;ld_ref) == 0 ) failed: lu is ffff88010cf8a000&lt;br/&gt;
&amp;lt;0&amp;gt;LustreError: 2659:0:(lod_dev.c:1493:lod_device_free()) LBUG&lt;br/&gt;
&amp;lt;4&amp;gt;Pid: 2659, comm: obd_zombid&lt;br/&gt;
&amp;lt;4&amp;gt;&lt;br/&gt;
&amp;lt;4&amp;gt;Call Trace:&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa02d6875&amp;gt;&amp;#93;&lt;/span&gt; libcfs_debug_dumpstack+0x55/0x80 &lt;span class=&quot;error&quot;&gt;&amp;#91;libcfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa02d6e77&amp;gt;&amp;#93;&lt;/span&gt; lbug_with_loc+0x47/0xb0 &lt;span class=&quot;error&quot;&gt;&amp;#91;libcfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa0eda121&amp;gt;&amp;#93;&lt;/span&gt; lod_device_free+0x2c1/0x330 &lt;span class=&quot;error&quot;&gt;&amp;#91;lod&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03ef9bd&amp;gt;&amp;#93;&lt;/span&gt; class_decref+0x3ed/0x4d0 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03d9afc&amp;gt;&amp;#93;&lt;/span&gt; obd_zombie_impexp_cull+0x61c/0xac0 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03da005&amp;gt;&amp;#93;&lt;/span&gt; obd_zombie_impexp_thread+0x65/0x190 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff81061d00&amp;gt;&amp;#93;&lt;/span&gt; ? default_wake_function+0x0/0x20&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03d9fa0&amp;gt;&amp;#93;&lt;/span&gt; ? obd_zombie_impexp_thread+0x0/0x190 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109abf6&amp;gt;&amp;#93;&lt;/span&gt; kthread+0x96/0xa0&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8100c20a&amp;gt;&amp;#93;&lt;/span&gt; child_rip+0xa/0x20&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109ab60&amp;gt;&amp;#93;&lt;/span&gt; ? kthread+0x0/0xa0&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8100c200&amp;gt;&amp;#93;&lt;/span&gt; ? child_rip+0x0/0x20&lt;br/&gt;
&amp;lt;4&amp;gt;&lt;br/&gt;
&amp;lt;0&amp;gt;Kernel panic - not syncing: LBUG&lt;br/&gt;
&amp;lt;4&amp;gt;Pid: 2659, comm: obd_zombid Not tainted 2.6.32-431.23.3.el6_lustre.x86_64 #1&lt;br/&gt;
&amp;lt;4&amp;gt;Call Trace:&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8152896c&amp;gt;&amp;#93;&lt;/span&gt; ? panic+0xa7/0x16f&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa02d6ecb&amp;gt;&amp;#93;&lt;/span&gt; ? lbug_with_loc+0x9b/0xb0 &lt;span class=&quot;error&quot;&gt;&amp;#91;libcfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa0eda121&amp;gt;&amp;#93;&lt;/span&gt; ? lod_device_free+0x2c1/0x330 &lt;span class=&quot;error&quot;&gt;&amp;#91;lod&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03ef9bd&amp;gt;&amp;#93;&lt;/span&gt; ? class_decref+0x3ed/0x4d0 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03d9afc&amp;gt;&amp;#93;&lt;/span&gt; ? obd_zombie_impexp_cull+0x61c/0xac0 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03da005&amp;gt;&amp;#93;&lt;/span&gt; ? obd_zombie_impexp_thread+0x65/0x190 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff81061d00&amp;gt;&amp;#93;&lt;/span&gt; ? default_wake_function+0x0/0x20&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa03d9fa0&amp;gt;&amp;#93;&lt;/span&gt; ? obd_zombie_impexp_thread+0x0/0x190 &lt;span class=&quot;error&quot;&gt;&amp;#91;obdclass&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109abf6&amp;gt;&amp;#93;&lt;/span&gt; ? kthread+0x96/0xa0&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8100c20a&amp;gt;&amp;#93;&lt;/span&gt; ? child_rip+0xa/0x20&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109ab60&amp;gt;&amp;#93;&lt;/span&gt; ? kthread+0x0/0xa0&lt;br/&gt;
&amp;lt;4&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8100c200&amp;gt;&amp;#93;&lt;/span&gt; ? child_rip+0x0/0x20&lt;/p&gt;</description>
                <environment></environment>
        <key id="32209">LU-7184</key>
            <summary>(lod_dev.c:1493:lod_device_free()) ASSERTION( atomic_read(&amp;lu-&gt;ld_ref) == 0 ) failed: lu is ffff88010cf8a000</summary>
                <type id="1" iconUrl="https://jira.whamcloud.com/secure/viewavatar?size=xsmall&amp;avatarId=11303&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.whamcloud.com/images/icons/priorities/major.svg">Major</priority>
                        <status id="5" iconUrl="https://jira.whamcloud.com/images/icons/statuses/resolved.png" description="A resolution has been taken, and it is awaiting verification by reporter. From here issues are either reopened, or are closed.">Resolved</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="1">Fixed</resolution>
                                        <assignee username="jhammond">John Hammond</assignee>
                                    <reporter username="jfilizetti">Jeremy Filizetti</reporter>
                        <labels>
                            <label>SSK</label>
                            <label>kerberos</label>
                    </labels>
                <created>Fri, 18 Sep 2015 01:33:52 +0000</created>
                <updated>Wed, 1 Jun 2016 22:23:07 +0000</updated>
                            <resolved>Wed, 14 Oct 2015 14:24:07 +0000</resolved>
                                    <version>Lustre 2.8.0</version>
                                    <fixVersion>Lustre 2.8.0</fixVersion>
                                        <due></due>
                            <votes>0</votes>
                                    <watches>9</watches>
                                                                            <comments>
                            <comment id="127734" author="jfilizetti" created="Fri, 18 Sep 2015 01:39:55 +0000"  >&lt;p&gt;Forgot to mention counter for ld_ref is 2:&lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;crash&amp;gt; struct lu_device 0xffff88010cf8a000
struct lu_device {
  ld_ref = {
    counter = 2
  }, 
  ld_type = 0xffffffffa0f12440, 
  ld_ops = 0xffffffffa0f08e40, 
  ld_site = 0xffff880118546098, 
  ld_proc_entry = 0x0, 
  ld_obd = 0xffff88011a186038, 
  ld_reference = {&amp;lt;No data fields&amp;gt;}, 
  ld_linkage = {
    next = 0xffff88010cf8a030, 
    prev = 0xffff88010cf8a030
  }
}
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                            <comment id="127817" author="green" created="Fri, 18 Sep 2015 17:34:26 +0000"  >&lt;p&gt;So it looks like error path for either key or in osp_obd_connect or somewhere along the call chain forgets to release a reference to lu device.&lt;/p&gt;

&lt;p&gt;Somebody need to go through there and fidn the place and add the decref that is missing.&lt;/p&gt;</comment>
                            <comment id="127820" author="jgmitter" created="Fri, 18 Sep 2015 17:37:41 +0000"  >&lt;p&gt;Hi John,&lt;br/&gt;
Can you take a look at the one?&lt;br/&gt;
Thanks.&lt;br/&gt;
Joe&lt;/p&gt;</comment>
                            <comment id="127999" author="jhammond" created="Mon, 21 Sep 2015 20:09:33 +0000"  >&lt;p&gt;In progress. The references are from opd_last_used_oid_file and opd_last_used_seq_file.&lt;/p&gt;</comment>
                            <comment id="128153" author="jhammond" created="Tue, 22 Sep 2015 19:45:04 +0000"  >&lt;p&gt;Di, during MDT mount if &lt;tt&gt;osp_init()&lt;/tt&gt; succeeds but &lt;tt&gt;lod_add_device()&lt;/tt&gt; fails before adding the OSP device to the LOD then we hit this since the OSP device still holds references to two objects from the MDT site (opd_last_used_oid_file and opd_last_used_seq_file). Can the finding/creation of these two objects be moved out of &lt;tt&gt;osp_init0()&lt;/tt&gt; and into some function later in the setup path?&lt;/p&gt;</comment>
                            <comment id="128161" author="di.wang" created="Tue, 22 Sep 2015 20:42:25 +0000"  >&lt;p&gt;It looks like osp_shutdown() is not being called in this case, since the OSP is not being added successfully. So it seems not just osp_last_used_fini(),  neither osp_sync_fini() nor osp_precreate_fini() are executed either.  So How about call ldo_process_config(env, next, CLEANUP);  in lod_add_device() error handler path? &lt;/p&gt;</comment>
                            <comment id="128424" author="gerrit" created="Thu, 24 Sep 2015 20:49:41 +0000"  >&lt;p&gt;John L. Hammond (john.hammond@intel.com) uploaded a new patch: &lt;a href=&quot;http://review.whamcloud.com/16635&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/16635&lt;/a&gt;&lt;br/&gt;
Subject: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-7184&quot; title=&quot;(lod_dev.c:1493:lod_device_free()) ASSERTION( atomic_read(&amp;amp;lu-&amp;gt;ld_ref) == 0 ) failed: lu is ffff88010cf8a000&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-7184&quot;&gt;&lt;del&gt;LU-7184&lt;/del&gt;&lt;/a&gt; lod: cleanup unused OSP devices on error&lt;br/&gt;
Project: fs/lustre-release&lt;br/&gt;
Branch: master&lt;br/&gt;
Current Patch Set: 1&lt;br/&gt;
Commit: 415bf6d50231dc8e804b6ddfa6b0aa2c2a5c92b1&lt;/p&gt;</comment>
                            <comment id="130334" author="gerrit" created="Wed, 14 Oct 2015 04:49:03 +0000"  >&lt;p&gt;Oleg Drokin (oleg.drokin@intel.com) merged in patch &lt;a href=&quot;http://review.whamcloud.com/16635/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://review.whamcloud.com/16635/&lt;/a&gt;&lt;br/&gt;
Subject: &lt;a href=&quot;https://jira.whamcloud.com/browse/LU-7184&quot; title=&quot;(lod_dev.c:1493:lod_device_free()) ASSERTION( atomic_read(&amp;amp;lu-&amp;gt;ld_ref) == 0 ) failed: lu is ffff88010cf8a000&quot; class=&quot;issue-link&quot; data-issue-key=&quot;LU-7184&quot;&gt;&lt;del&gt;LU-7184&lt;/del&gt;&lt;/a&gt; lod: cleanup unused OSP devices on error&lt;br/&gt;
Project: fs/lustre-release&lt;br/&gt;
Branch: master&lt;br/&gt;
Current Patch Set: &lt;br/&gt;
Commit: d2d725d2e2d31899f0453c967f5707a72e796fa0&lt;/p&gt;</comment>
                            <comment id="130372" author="jgmitter" created="Wed, 14 Oct 2015 14:24:07 +0000"  >&lt;p&gt;Landed for 2.8&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Related</name>
                                            <outwardlinks description="is related to ">
                                        <issuelink>
            <issuekey id="18738">LU-3289</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="33585">LU-7546</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                            <customfield id="customfield_10890" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10390" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hzxo2f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10090" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10060" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Severity</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10022"><![CDATA[3]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        </customfields>
    </item>
</channel>
</rss>