[#LU-147] avoid 8k obd device amount limit

The patch is incomplete. Here are some work remaining:
1. We have to make sure that the obd devices have at least 1 refcount to stay in obd_dev_list and hash list, this means class_attach() should be modified;
2. pull the freeing obd device out of uuid_hash, etc earlier, better in class_detach;
3. the patch is not even compiled.
diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 8b00ad9..c8bfbf5 100644
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -509,36 +509,40 @@ const char *obd_dev_status(struct obd_device *obd)
         return status;
 }
 
-#define cfs_list_entry_safe(pos, head, type, member)  \
-        (pos == head ? NULL : cfs_list_entry(pos, type, member))
-
-void obd_devlist_first(struct obd_device **pos)
+struct obd_device *obd_devlist_first(void)
 {
         struct obd_device *obd;
+        struct obd_device *founc = NULL;
 
         cfs_spin_lock(&obd_dev_lock);
-        obd = cfs_list_entry_safe(obd_dev_list.next, &obd_dev_list,
-                                  struct obd_device, obd_list);
-        if (obd != NULL)
-                class_incref(obd, "devlist", obd);
+        cfs_list_for_each_entry(obd, &obd_dev_list, obd_list) {
+                if (cfs_atomic_read(&obd->obd_refcount) > 0) {
+                        found = obd;
+                        class_incref(obd, "devlist", obd);
+                        break;
+                }
+        }
         cfs_spin_unlock(&obd_dev_lock);
-
-        *pos = obd;
+        return found;
 }
 
-void obd_devlist_next(struct obd_device **pos)
+struct obd_device *obd_devlist_next(struct obd_device *obd)
 {
-        struct obd_device *obd = NULL;
+        struct obd_device *tmp   = obd;
+        struct obd_device *found = NULL;
 
         cfs_spin_lock(&obd_dev_lock);
-        obd = cfs_list_entry_safe((*pos)->obd_list.next, &obd_dev_list,
-                                  struct obd_device, obd_list);
-        if (obd)
-                class_incref(obd, "devlist", obd);
+        cfs_list_for_each_entry_continue(tmp, &obd_dev_list, obd_list) {
+                if (cfs_atomic_read(&tmp->obd_refcount) > 0) {
+                        found = tmp;
+                        class_incref(tmp, "devlist", tmp);
+                        break;
+                }
+        }
         cfs_spin_unlock(&obd_dev_lock);
 
-        class_decref(*pos, "devlist", *pos);
-        *pos = obd;
+        class_decref(obd, "devlist", obd);
+        return found;
 }
 
 void obd_devlist_last(struct obd_device *pos)
@@ -694,10 +698,9 @@ void class_obd_list(void)
         const char *status;
         struct obd_device *obd;
 
-        for (obd_devlist_first(&obd);
+        for (obd = obd_devlist_first();
              obd != NULL;
-             obd_devlist_next(&obd)) {
-
+             obd = obd_devlist_next(obd)) {
                 status = obd_dev_status(obd);
                 LCONSOLE(D_CONFIG, "%3d %s %s %s %s %d\n",
                          obd->obd_minor, status, obd->obd_type->typ_name,
@@ -710,29 +713,28 @@ void class_obd_list(void)
 /* Search for a client OBD connected to tgt_uuid.  If grp_uuid is
    specified, then only the client with that uuid is returned,
    otherwise any client connected to the tgt is returned. */
-struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid,
-                                          const char * typ_name,
-                                          struct obd_uuid *grp_uuid)
+struct obd_device *class_find_client_obd(struct obd_uuid *tgt_uuid,
+                                         const char *name,
+                                         struct obd_uuid *guuid)
 {
         struct obd_device *obd;
+        struct obd_device *found = NULL;
 
-        for (obd_devlist_first(&obd);
+        for (obd = obd_devlist_first();
              obd != NULL;
-             obd_devlist_next(&obd)) {
+             obd = obd_devlist_next(obd)) {
                 /* XXX per type list ? */
-                if ((strncmp(obd->obd_type->typ_name, typ_name,
-                             strlen(typ_name)) == 0)) {
-                        if (obd_uuid_equals(tgt_uuid,
-                                            &obd->u.cli.cl_target_uuid) &&
-                            ((grp_uuid)? obd_uuid_equals(grp_uuid,
-                                                         &obd->obd_uuid) : 1)) {
-                                obd_devlist_last(obd);
-                                return obd;
-                        }
+                if (strncmp(obd->obd_type->typ_name, name, strlen(name)))
+                        continue;
+
+                if (obd_uuid_equals(tgt_uuid, &obd->u.cli.cl_target_uuid) &&
+                    (!guuid || obd_uuid_equals(guuid, &obd->obd_uuid))) {
+                        found = obd;
+                        break;
                 }
         }
 
-        return NULL;
+        return found;
 }
 
 /* Iterate the obd_device list looking devices have grp_uuid. Start
diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c
index 6b493c7..f44175b 100644
--- a/lustre/obdclass/obd_config.c
+++ b/lustre/obdclass/obd_config.c
@@ -579,17 +579,10 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
 
         LASSERT(obd->obd_self_export);
 
-        /* The three references that should be remaining are the
-         * obd_self_export and the attach and setup references. */
-        if (cfs_atomic_read(&obd->obd_refcount) > 3) {
-                /* refcounf - 3 might be the number of real exports
-                   (excluding self export). But class_incref is called
-                   by other things as well, so don't count on it. */
-                CDEBUG(D_IOCTL, "%s: forcing exports to disconnect: %d\n",
-                       obd->obd_name, cfs_atomic_read(&obd->obd_refcount) - 3);
-                dump_exports(obd, 0);
-                class_disconnect_exports(obd);
-        }
+        CDEBUG(D_IOCTL, "%s: forcing exports to disconnect: %d\n",
+               obd->obd_name, cfs_atomic_read(&obd->obd_refcount) - 3);
+        dump_exports(obd, 0);
+        class_disconnect_exports(obd);
 
         /* destroy an uuid-export hash body */
         if (obd->obd_uuid_hash) {
@@ -614,6 +607,20 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
         if (err)
                 CERROR("Precleanup %s returned %d\n",
                        obd->obd_name, err);
+
+        /* cleanup self export */
+        if (obd->obd_self_export) {
+                /* All exports have been destroyed; there should
+                   be no more in-progress ops by this point.*/
+                cfs_spin_lock(&obd->obd_self_export->exp_lock);
                                                                                                                       
+                obd->obd_self_export->exp_flags |= exp_flags_from_obd(obd);
+                cfs_spin_unlock(&obd->obd_self_export->exp_lock);
+
+                /* note that we'll recurse into class_decref again */
+                class_unlink_export(obd->obd_self_export);
+                obd->obd_self_export = NULL;
+        }
+
         class_decref(obd, "setup", obd);
         obd->obd_set_up = 0;
         RETURN(0);
@@ -622,10 +629,12 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
 struct obd_device *class_incref(struct obd_device *obd,
                                 const char *scope, const void *source)
 {
+        LASSERT(cfs_atomic_read(&obd->obd_refcount) > 0);
         lu_ref_add_atomic(&obd->obd_reference, scope, source);
         cfs_atomic_inc(&obd->obd_refcount);
-        CDEBUG(D_INFO, "incref %s (%p) now %d\n", obd->obd_name, obd,
-               cfs_atomic_read(&obd->obd_refcount));
+
+        CDEBUG(D_INFO, "incref %s (%p) at %s:%s, now %d\n", obd->obd_name, obd,
+               scope, source, cfs_atomic_read(&obd->obd_refcount));
 
         return obd;
 }
@@ -633,29 +642,20 @@ struct obd_device *class_incref(struct obd_device *obd,
 void class_decref(struct obd_device *obd, const char *scope, const void *source)
 {
         int err;
-        int refs;
 
-        refs = cfs_atomic_dec_return(&obd->obd_refcount);
-        lu_ref_del(&obd->obd_reference, scope, source);
-
-        CDEBUG(D_INFO, "Decref %s (%p) now %d\n", obd->obd_name, obd, refs);
-
-        if ((refs == 1) && obd->obd_stopping) {
-                /* All exports have been destroyed; there should
-                   be no more in-progress ops by this point.*/
-
-                cfs_spin_lock(&obd->obd_self_export->exp_lock);
-                obd->obd_self_export->exp_flags |= exp_flags_from_obd(obd);
-                cfs_spin_unlock(&obd->obd_self_export->exp_lock);
+        CDEBUG(D_INFO, "decref %s (%p) at %s:%s now %d\n", obd->obd_name, obd,
+               scope, source, cfs_atomic_read(&obd->obd_refcount));
-                /* note that we'll recurse into class_decref again */
-                class_unlink_export(obd->obd_self_export);
-                return;
-        }
+        LASSERT(cfs_atomic_read(&obd->obd_refcount) > 0);
+        lu_ref_del(&obd->obd_reference, scope, source);
+        if (cfs_atomic_dec_and_lock(&obd->obd_refcount, &obd_dev_lock)) {
+                cfs_list_del(&obd->obd_list);
+                cfs_spin_unlock(&obd_dev_lock);
 
-        if (refs == 0) {
                 CDEBUG(D_CONFIG, "finishing cleanup of obd %s (%s)\n",
                        obd->obd_name, obd->obd_uuid.uuid);
+
+                LASSERT(obd->obd_self_export == NULL);
                 LASSERT(!obd->obd_attached);
                 if (obd->obd_stopping) {
                         /* If we're not stopping, we were never set up */
[LU-147] avoid 8k obd device amount limit Created: 18/Mar/11 Updated: 22/Jan/24 Resolved: 22/Jan/24
Status:	Resolved
Project:	Lustre
Component/s:	None
Affects Version/s:	Lustre 2.1.0
Fix Version/s:	Lustre 2.1.0
[LU-147] avoid 8k obd device amount limit Created: 18/Mar/11 Updated: 22/Jan/24 Resolved: 22/Jan/24
