[#LU-1013] recovery-mds lu_object.c:116:lu_object_put()) ASSERTION(cfs_list_empty(&top->loh

I have met similar issues in my branch for OI Scrub and blocked my OI Scrub test. I found there is a race condition in lu_object_find_try() and lu_object_put(). For example:

static struct lu_object *lu_object_find_try(const struct lu_env *env,
                                            struct lu_device *dev,
                                            const struct lu_fid *f,
                                            const struct lu_object_conf *conf,
                                            cfs_waitlink_t *waiter)
{
...
(step1)        o = lu_object_alloc(env, dev, f, conf);
        if (unlikely(IS_ERR(o)))
                return o;

        LASSERT(lu_fid_eq(lu_object_fid(o), f));

        cfs_hash_bd_lock(hs, &bd, 1);

        shadow = htable_lookup(s, &bd, f, waiter, &version);
        if (likely(shadow == NULL)) {
                struct lu_site_bkt_data *bkt;

                bkt = cfs_hash_bd_extra_get(hs, &bd);
                cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
                bkt->lsb_busy++;
                cfs_hash_bd_unlock(hs, &bd, 1);
(step2)                return o;
        }

        lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_RACE);
        cfs_hash_bd_unlock(hs, &bd, 1);
        lu_object_free(env, o);
(step3)        return shadow;
}

void lu_object_put(const struct lu_env *env, struct lu_object *o)
{
...
        if (!lu_object_is_dying(top)) {
(step4)                LASSERT(cfs_list_empty(&top->loh_lru));
(step5)                cfs_list_add_tail(&top->loh_lru, &bkt->lsb_lru);
                cfs_hash_bd_unlock(site->ls_obj_hash, &bd, 1);
                return;
        }
...
}

Thread1 and Thread2 try to find some object with the same FID concurrently, and the object is not allocated in memory yet. Consider the following sequence:

1) Thread1 step1
2) Thread2 step1
3) Thread1 step2
4) Thread1 step4
5) Thread1 step5
6) Thread2 step3
7) Thread2 step4

mds kernel: LustreError: 28264:0:(lu_object.c:116:lu_object_put()) ASSERTION(cfs_list_empty(&top->loh_lru)) failed

I have made following patch to fix the race, if you do not mind, I will push the patch to gerrit for review.

===========================
diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c
index 2ad22f0..f26c534 100644
--- a/lustre/obdclass/lu_object.c
+++ b/lustre/obdclass/lu_object.c
@@ -619,7 +624,7 @@ static struct lu_object *lu_object_find_try(const struct lu_env *env,
         cfs_hash_bd_lock(hs, &bd, 1);

         shadow = htable_lookup(s, &bd, f, waiter, &version);
-        if (likely(shadow == NULL)) {
+        if (shadow == NULL) {
                 struct lu_site_bkt_data *bkt;

                 bkt = cfs_hash_bd_extra_get(hs, &bd);
@@ -627,12 +632,14 @@ static struct lu_object *lu_object_find_try(const struct lu_env *env,
                 bkt->lsb_busy++;
                 cfs_hash_bd_unlock(hs, &bd, 1);
                 return o;
+        } else {
+                if (!cfs_list_empty(&shadow->lo_header->loh_lru))
+                        cfs_list_del_init(&shadow->lo_header->loh_lru);
+                lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_RACE);
+                cfs_hash_bd_unlock(hs, &bd, 1);
+                lu_object_free(env, o);
+                return shadow;
         }
-
-        lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_RACE);
-        cfs_hash_bd_unlock(hs, &bd, 1);
-        lu_object_free(env, o);
-        return shadow;
 }

 /**
===========================

[LU-1013] recovery-mds lu_object.c:116:lu_object_put()) ASSERTION(cfs_list_empty(&top->loh_lru)) failed Created: 19/Jan/12 Updated: 27/Mar/12 Resolved: 13/Feb/12
Status:	Resolved
Project:	Lustre
Component/s:	None
Affects Version/s:	Lustre 2.2.0
Fix Version/s:	None

[LU-1013] recovery-mds lu_object.c:116:lu_object_put()) ASSERTION(cfs_list_empty(&top->loh_lru)) failed Created: 19/Jan/12 Updated: 27/Mar/12 Resolved: 13/Feb/12