[#LU-8562] osp_precreate_cleanup_orphans/osp_precreate

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index c64ebab..f5026dc 100755 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -6796,6 +6796,32 @@ test_97() { } run_test 97 "ldev returns correct ouput when querying based on role" +test_98() { + local_mode || { skip "Need single node setup"; return; } + local cmp=0 + local dev=$FSNAME-OST0000-osc-MDT0000 + setupall + + createmany -o $DIR1/$tfile-%d 50000& + cmp=$! + # MDT->OST reconnection causes MDT<->OST last_id synchornisation + # via osp_precreate_cleanup_orphans. + for i in $(seq 0 100); do + for k in $(seq 0 10); do + $LCTL --device $dev deactivate + $LCTL --device $dev activate + done + ls -asl $MOUNT | grep '???' && \ + (kill -9 $cmp &>/dev/null; \ + error "File hasn't object on OST") + ps -A -o pid | grep $cmp 1>/dev/null || break + done + wait $cmp + stopall +} +run_test 98 "Race MDT->OST reconnection with create" + +

I was testing out patch 22211 and (if my understanding is correct) may have found a defect.

It seems osp_precreate_thread() can get stuck because d->opd_got_disconnected never gets reset. When opd_got_disconnected is set, osp_precreate_cleanup_orphans() returns early with EAGAIN and can't clear d->opd_pre_recovering. And because d->opd_pre_recovering can't be cleared we always hit the break statement below and don't clear d->opd_got_disconnected. So osp_precreate_cleanup_orphans() is stuck always failing.

        while (osp_precreate_running(d)) {
                /*
                 * need to be connected to OST
                 */
                while (osp_precreate_running(d)) {
+                       if (d->opd_pre_recovering &&
+                           d->opd_imp_connected)
+                               break;
                        l_wait_event(d->opd_pre_waitq,
                                     !osp_precreate_running(d) ||
                                     d->opd_new_connection,
                                     &lwi);
 
                        if (!d->opd_new_connection)
                                continue;
 
                        d->opd_new_connection = 0;
                        d->opd_got_disconnected = 0;
                        break;
                }
 
                if (!osp_precreate_running(d))
                        break;
 
                LASSERT(d->opd_obd->u.cli.cl_seq != NULL);
                /* Sigh, fid client is not ready yet */
                if (d->opd_obd->u.cli.cl_seq->lcs_exp == NULL)
                        continue;
 
                /* Init fid for osp_precreate if necessary */
                rc = osp_init_pre_fid(d);
                if (rc != 0) {
                        class_export_put(d->opd_exp);
                        d->opd_obd->u.cli.cl_seq->lcs_exp = NULL;
                        CERROR("%s: init pre fid error: rc = %d\n",
                               d->opd_obd->obd_name, rc);
                        continue;
                }
 
                osp_statfs_update(d);
 
                /*
                 * Clean up orphans or recreate missing objects.
                 */
                rc = osp_precreate_cleanup_orphans(&env, d);
-               if (rc != 0)
+               if (rc != 0) {
+                       schedule_timeout_interruptible(cfs_time_seconds(1));
                        continue;
+               }
                /*
                 * connected, can handle precreates now
                 */

[LU-8562] osp_precreate_cleanup_orphans/osp_precreate_reserve race may cause data loss Created: 29/Aug/16 Updated: 16/Jul/19 Resolved: 16/Feb/17
Status:	Resolved
Project:	Lustre
Component/s:	None
Affects Version/s:	Lustre 2.8.0, Lustre 2.9.0
Fix Version/s:	Lustre 2.10.0

[LU-8562] osp_precreate_cleanup_orphans/osp_precreate_reserve race may cause data loss Created: 29/Aug/16 Updated: 16/Jul/19 Resolved: 16/Feb/17