From 485b5a7e830caeb0bab3d56e0c1228d4f58a97fc Mon Sep 17 00:00:00 2001
From: root <root@opahsx121.cornelisnetworks.com>
Date: Thu, 19 Jan 2023 08:20:43 +0100
Subject: [PATCH 1/5] Programatically force a map-unmap without a post.

---
 lnet/klnds/o2iblnd/o2iblnd_cb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index ca408378ab..cc0b7c00bc 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -610,6 +610,10 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx,
 #endif
 
 	fps = net->ibn_fmr_ps[cpt];
+{
+static unsigned int tx_count;
+
+again:
 	rc = kiblnd_fmr_pool_map(fps, tx, rd, nob, 0, &tx->tx_fmr);
 	if (rc != 0) {
 		CERROR("Can't map %u bytes (%u/%u)s: %d\n", nob,
@@ -617,6 +621,15 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx,
 		return rc;
 	}
 
+	if (tx_count++ == 100) {
+		printk("%s: forcing map-unmap on mr 0x%lx\n",
+			__func__,
+			(unsigned long)tx->tx_fmr.fmr_frd->frd_mr);
+		// the status argument is not used for FRWR
+		kiblnd_fmr_pool_unmap(&tx->tx_fmr, 0);
+		goto again;
+	}
+}
 	/*
 	 * If rd is not tx_rd, it's going to get sent to a peer_ni, who will
 	 * need the rkey
-- 
2.39.1

From 574c638f664c9f706a0e1816c46cc3fce905d727 Mon Sep 17 00:00:00 2001
From: Dean Luick <dean.luick@cornelisnetworks.com>
Date: Thu, 19 Jan 2023 08:21:08 +0100
Subject: [PATCH 2/5] Debugging to test the no-post theory

"No-post theory" = Pool mr was mapped then unmapped without being
posted.  This will result in the ib key being permanently skewed
from the rdmavt key.
---
 lnet/klnds/o2iblnd/o2iblnd.c    | 4 ++++
 lnet/klnds/o2iblnd/o2iblnd_cb.c | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 8a14b86f90..738e289b1b 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1822,6 +1822,10 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 		struct kib_fast_reg_descriptor *frd = fmr->fmr_frd;
 
 		if (frd) {
+			if (!frd->frd_posted) {
+				printk("%s: mr 0x%lx not posted\n", __func__,
+					(unsigned long)frd->frd_mr);
+			}
 			frd->frd_valid = false;
 			frd->frd_posted = false;
 			fmr->fmr_frd = NULL;
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index cc0b7c00bc..6b19b7ba77 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1014,6 +1014,13 @@ __must_hold(&conn->ibc_lock)
 #else
 			rc = ib_post_send(conn->ibc_cmid->qp, wr, &bad);
 #endif
+		if (rc && frd) {
+			printk("%s: mr 0x%lx, posted %d, err %d, send failure\n",
+			       __func__,
+			       (unsigned long)frd->frd_mr,
+			       (int)frd->frd_posted,
+			       rc);
+		}
 	}
 
 	conn->ibc_last_send = ktime_get();
-- 
2.39.1

From 37250a74313330f5f6a5c608cf992c9de023089c Mon Sep 17 00:00:00 2001
From: root <root@opahsx121.cornelisnetworks.com>
Date: Thu, 19 Jan 2023 08:21:14 +0100
Subject: [PATCH 3/5] Extra debug for o2iblind

---
 lnet/klnds/o2iblnd/o2iblnd.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 738e289b1b..a271b0a67a 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1820,6 +1820,12 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 #endif /* HAVE_FMR_POOL_API */
 	{
 		struct kib_fast_reg_descriptor *frd = fmr->fmr_frd;
+		if (frd) {
+			struct ib_mr *mr = frd->frd_mr;
+			printk("%s: mr 0x%lx, rkey 0x%08x, posted %d\n",
+				__func__, (unsigned long)mr, mr->rkey,
+				(int)frd->frd_posted);
+		}
 
 		if (frd) {
 			if (!frd->frd_posted) {
@@ -1902,6 +1908,7 @@ again:
 		} else
 #endif /* HAVE_FMR_POOL_API */
 		{
+static unsigned int count;
 			if (!list_empty(&fpo->fast_reg.fpo_pool_list)) {
 				struct kib_fast_reg_descriptor *frd;
 #ifdef HAVE_IB_MAP_MR_SG
@@ -1923,6 +1930,13 @@ again:
 				frpl = frd->frd_frpl;
 #endif
 				mr   = frd->frd_mr;
+				printk("%s  : mr 0x%lx, rkey 0x%08x, is_rx %d, count %u\n", __func__, (unsigned long)mr, mr->rkey, (int)is_rx, count);
+#if 0
+				if ((count % 100) == 0) {
+					dump_stack();
+				}
+#endif
+				count++;
 
 				if (!frd->frd_valid) {
 					struct ib_rdma_wr *inv_wr;
@@ -2610,6 +2624,7 @@ kiblnd_hdev_get_attr(struct kib_hca_dev *hdev)
 #endif /* HAVE_FMR_POOL_API */
 	if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
 		LCONSOLE_INFO("Using FastReg for registration\n");
+		LCONSOLE_INFO("CN debug v3\n");
 		hdev->ibh_dev->ibd_dev_caps |= IBLND_DEV_CAPS_FASTREG_ENABLED;
 #ifndef HAVE_IB_ALLOC_FAST_REG_MR
 #ifdef IB_DEVICE_SG_GAPS_REG
-- 
2.39.1

From 55dbd75caca194f5279b095069b6a9cb28cf2e79 Mon Sep 17 00:00:00 2001
From: Dean Luick <dean.luick@cornelisnetworks.com>
Date: Thu, 19 Jan 2023 08:21:29 +0100
Subject: [PATCH 4/5] o2iblnd: Fix key mismatch issue

If a pool memory region (mr) is mapped then unmapped without being
used, its key becomes out of sync with the RDMA subsystem.

At pool mr map time, the present code will create a local
invalidate work request (wr) using the mr's present key and then
change the mr's key.  When the mr is first used after being mapped,
the local invalidate wr will invalidate the original mr key, and
then a fast register wr is used with the modified key.  The fast
register will update the RDMA subsystem's key for the mr.

The error occurs when the mr is never used.  The next time the mr
is mapped, a local invalidate wr will again be created, but this
time it will use the mr's modified key.  The RDMA subsystem never
saw the original local invalidate, so now the RDMA subsystem's
key for the mr and o2iblnd's key for the mr are out of sync.

Fix the issue by tracking if the invalidate has been used.
Repurpose the boolean frd->frd_valid.  Presently, frd_valid is
always false.  Remove the code that used frd_valid to conditionally
split the invalidate from the fast register.  Instead, use frd_valid
to indicate when a new invalidate needs to be generated.  After a
post, evaluate if the invalidate was successfully used in the post.

These changes are only meaningful to the FRWR code path.  The failure
has only been observed when using Omni-Path Architecture.
---
 lnet/klnds/o2iblnd/o2iblnd.c    |  6 ++----
 lnet/klnds/o2iblnd/o2iblnd_cb.c | 16 ++++++++++------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index a271b0a67a..458ce1bfd9 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1626,9 +1626,7 @@ static int kiblnd_alloc_freg_pool(struct kib_fmr_poolset *fps,
 			goto out_middle;
 		}
 
-		/* There appears to be a bug in MLX5 code where you must
-		 * invalidate the rkey of a new FastReg pool before first
-		 * using it. Thus, I am marking the FRD invalid here. */
+		/* indicate that the local invalidate needs to be generated */
 		frd->frd_valid = false;
 
 		list_add_tail(&frd->frd_list, &fpo->fast_reg.fpo_pool_list);
@@ -1832,7 +1830,6 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 				printk("%s: mr 0x%lx not posted\n", __func__,
 					(unsigned long)frd->frd_mr);
 			}
-			frd->frd_valid = false;
 			frd->frd_posted = false;
 			fmr->fmr_frd = NULL;
 			spin_lock(&fps->fps_lock);
@@ -1942,6 +1939,7 @@ static unsigned int count;
 					struct ib_rdma_wr *inv_wr;
 					__u32 key = is_rx ? mr->rkey : mr->lkey;
 
+					frd->frd_valid = true;
 					inv_wr = &frd->frd_inv_wr;
 					memset(inv_wr, 0, sizeof(*inv_wr));
 
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 6b19b7ba77..c42231c82c 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -990,12 +990,8 @@ __must_hold(&conn->ibc_lock)
 		struct ib_send_wr *wr  = &tx->tx_wrq[0].wr;
 
 		if (frd != NULL && !frd->frd_posted) {
-			if (!frd->frd_valid) {
-				wr = &frd->frd_inv_wr.wr;
-				wr->next = &frd->frd_fastreg_wr.wr;
-			} else {
-				wr = &frd->frd_fastreg_wr.wr;
-			}
+			wr = &frd->frd_inv_wr.wr;
+			wr->next = &frd->frd_fastreg_wr.wr;
 			frd->frd_fastreg_wr.wr.next = &tx->tx_wrq[0].wr;
 		}
 
@@ -1014,6 +1010,14 @@ __must_hold(&conn->ibc_lock)
 #else
 			rc = ib_post_send(conn->ibc_cmid->qp, wr, &bad);
 #endif
+		if (frd && !frd->frd_posted) {
+			/*
+			 * The local invalidate becomes invalid (has been
+			 * successfully used) if the post succeeds or the
+			 * failing wr was not the invalidate.
+			 */
+			frd->frd_valid = !(rc == 0 || (bad != &frd->frd_inv_wr.wr));
+		}
 		if (rc && frd) {
 			printk("%s: mr 0x%lx, posted %d, err %d, send failure\n",
 			       __func__,
-- 
2.39.1

From 5a6a1683e9fc727e4bfb1be57fd6d982df953fd8 Mon Sep 17 00:00:00 2001
From: Serguei Smirnov <ssmirnov@whamcloud.com>
Date: Thu, 19 May 2022 17:20:14 -0700
Subject: [PATCH 5/5] LU-16172 o2iblnd: add verbose debug prints for rx/tx
 events

Added/modified debug messages for syncing with mlnx driver
debug output. On rx/tx events print message type, size and
peer credits.

Test-Parameters: fortestonly testlist=sanity-lnet
Signed-off-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Change-Id: If7c2de56d8e4ef71085c3b49caf589e2f3864b15
---
 lnet/klnds/o2iblnd/o2iblnd.c    |   2 +-
 lnet/klnds/o2iblnd/o2iblnd.h    |  38 ++++++-
 lnet/klnds/o2iblnd/o2iblnd_cb.c | 174 +++++++++++++++++++++-----------
 lnet/lnet/lib-move.c            |   4 +
 4 files changed, 152 insertions(+), 66 deletions(-)

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 458ce1bfd9..956581e359 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2265,7 +2265,7 @@ again:
 	CDEBUG(D_NET, "%s pool exhausted, allocate new pool\n", ps->ps_name);
 	time_before = ktime_get();
 	rc = ps->ps_pool_create(ps, ps->ps_pool_size, &pool);
-	CDEBUG(D_NET, "ps_pool_create took %lld ms to complete",
+	CDEBUG(D_NET, "ps_pool_create took %lld ms to complete\n",
 	       ktime_ms_delta(ktime_get(), time_before));
 
 	spin_lock(&ps->ps_lock);
diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 1098f789f6..fd5934cf2f 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -838,8 +838,6 @@ kiblnd_dev_can_failover(struct kib_dev *dev)
 
 #define kiblnd_conn_addref(conn)                                \
 do {                                                            \
-        CDEBUG(D_NET, "conn[%p] (%d)++\n",                      \
-	       (conn), atomic_read(&(conn)->ibc_refcount)); \
 	atomic_inc(&(conn)->ibc_refcount);                  \
 } while (0)
 
@@ -847,8 +845,6 @@ do {                                                            \
 do {									\
 	unsigned long flags;						\
 									\
-	CDEBUG(D_NET, "conn[%p] (%d)--\n",				\
-	       (conn), atomic_read(&(conn)->ibc_refcount));		\
 	LASSERT_ATOMIC_POS(&(conn)->ibc_refcount);			\
 	if (atomic_dec_and_test(&(conn)->ibc_refcount)) {		\
 		spin_lock_irqsave(&kiblnd_data.kib_connd_lock, flags);	\
@@ -1225,4 +1221,38 @@ int kiblnd_recv(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg,
 		int delayed, unsigned int niov, struct kvec *iov,
 		lnet_kiov_t *kiov, unsigned int offset, unsigned int mlen,
 		unsigned int rlen);
+unsigned int kiblnd_get_dev_prio(struct lnet_ni *ni, unsigned int dev_idx);
 
+static inline void kiblnd_dump_conn_dbg(unsigned int code, struct kib_conn *conn)
+{
+
+	if (!conn)
+		return;
+
+	if (conn->ibc_cmid)
+		CNETERR("%u conn %p state %d nposted %d/%d c/o/r %d/%d/%d ce %d : cm_id %p qp_num 0x%x device_name %s\n",
+			code,
+			conn,
+			conn->ibc_state,
+			conn->ibc_noops_posted,
+			conn->ibc_nsends_posted,
+			conn->ibc_credits,
+			conn->ibc_outstanding_credits,
+			conn->ibc_reserved_credits,
+			conn->ibc_comms_error,
+			conn->ibc_cmid,
+			conn->ibc_cmid->qp ? conn->ibc_cmid->qp->qp_num : 0,
+			conn->ibc_cmid->qp ? (conn->ibc_cmid->qp->device ? dev_name(&conn->ibc_cmid->qp->device->dev) : "NULL") : "NULL");
+	else
+		CNETERR("%u conn %p state %d nposted %d/%d c/o/r %d/%d/%d ce %d : cm_id NULL\n",
+			code,
+			conn,
+			conn->ibc_state,
+			conn->ibc_noops_posted,
+			conn->ibc_nsends_posted,
+			conn->ibc_credits,
+			conn->ibc_outstanding_credits,
+			conn->ibc_reserved_credits,
+			conn->ibc_comms_error
+			);
+}
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index c42231c82c..8b62ef7b3d 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -338,9 +338,16 @@ kiblnd_handle_rx(struct kib_rx *rx)
 
         LASSERT (conn->ibc_state >= IBLND_CONN_ESTABLISHED);
 
-        CDEBUG (D_NET, "Received %x[%d] from %s\n",
-                msg->ibm_type, credits,
-                libcfs_nid2str(conn->ibc_peer->ibp_nid));
+	CDEBUG(D_NET, "Received %x[%d] nob %u from %s conn %p c/o/r %d/%d/%d cm_id %p qp_num 0x%x\n",
+		msg->ibm_type, credits,
+		msg->ibm_nob,
+		libcfs_nid2str(conn->ibc_peer->ibp_nid),
+		conn,
+		conn->ibc_credits,
+		conn->ibc_outstanding_credits,
+		conn->ibc_reserved_credits,
+		conn->ibc_cmid,
+		conn->ibc_cmid->qp ? conn->ibc_cmid->qp->qp_num : 0);
 
         if (credits != 0) {
                 /* Have I received credits that will let me send? */
@@ -481,8 +488,8 @@ kiblnd_handle_rx(struct kib_rx *rx)
         if (rc < 0)                             /* protocol error */
                 kiblnd_close_conn(conn, rc);
 
-        if (post_credit != IBLND_POSTRX_DONT_POST)
-                kiblnd_post_rx(rx, post_credit);
+	if (post_credit != IBLND_POSTRX_DONT_POST)
+		kiblnd_post_rx(rx, post_credit);
 }
 
 static void
@@ -925,11 +932,17 @@ __must_hold(&conn->ibc_lock)
 		return -EAGAIN;
 	}
 
-        if (credit != 0 && conn->ibc_credits == 0) {   /* no credits */
-                CDEBUG(D_NET, "%s: no credits\n",
-                       libcfs_nid2str(peer_ni->ibp_nid));
-                return -EAGAIN;
-        }
+	if (credit != 0 && conn->ibc_credits == 0) {   /* no credits */
+		CDEBUG(D_NET, "%s: no credits conn %p c/o/r %d/%d/%d cm_id %p qp_num 0x%x\n",
+			libcfs_nid2str(peer_ni->ibp_nid),
+			conn,
+			conn->ibc_credits,
+			conn->ibc_outstanding_credits,
+			conn->ibc_reserved_credits,
+			conn->ibc_cmid,
+			conn->ibc_cmid->qp ? conn->ibc_cmid->qp->qp_num : 0);
+		return -EAGAIN;
+	}
 
         if (credit != 0 && !IBLND_OOB_CAPABLE(ver) &&
             conn->ibc_credits == 1 &&   /* last credit reserved */
@@ -954,11 +967,28 @@ __must_hold(&conn->ibc_lock)
 		tx->tx_hstatus = LNET_MSG_STATUS_LOCAL_ERROR;
 		kiblnd_tx_done(tx);
 		spin_lock(&conn->ibc_lock);
-                CDEBUG(D_NET, "%s(%d): redundant or enough NOOP\n",
-                       libcfs_nid2str(peer_ni->ibp_nid),
-                       conn->ibc_noops_posted);
-                return 0;
-        }
+		CDEBUG(D_NET, "%s(%d): redundant or enough NOOP conn %p c/o/r %d/%d/%d cm_id %p qp_num 0x%x\n",
+			libcfs_nid2str(peer_ni->ibp_nid),
+			conn->ibc_noops_posted,
+			conn,
+			conn->ibc_credits,
+			conn->ibc_outstanding_credits,
+			conn->ibc_reserved_credits,
+			conn->ibc_cmid,
+			conn->ibc_cmid->qp ? conn->ibc_cmid->qp->qp_num : 0);
+		return -EAGAIN;
+	}
+
+	CDEBUG(D_NET, "Transmit %x[%d] nob %u to %s conn %p c/o/r %d/%d/%d cm_id %p qp_num 0x%x\n",
+		msg->ibm_type, credit,
+		msg->ibm_nob,
+		libcfs_nid2str(conn->ibc_peer->ibp_nid),
+		conn,
+		conn->ibc_credits,
+		conn->ibc_outstanding_credits,
+		conn->ibc_reserved_credits,
+		conn->ibc_cmid,
+		conn->ibc_cmid->qp ? conn->ibc_cmid->qp->qp_num : 0);
 
         kiblnd_pack_msg(peer_ni->ibp_ni, msg, ver, conn->ibc_outstanding_credits,
                         peer_ni->ibp_nid, conn->ibc_incarnation);
@@ -1175,6 +1205,8 @@ kiblnd_tx_complete(struct kib_tx *tx, int status)
 		tx->tx_hstatus = LNET_MSG_STATUS_REMOTE_DROPPED;
                 tx->tx_waiting = 0;             /* don't wait for peer_ni */
                 tx->tx_status = -EIO;
+
+		kiblnd_dump_conn_dbg(0x03, conn);
         }
 
         idle = (tx->tx_sending == 0) &&         /* This is the final callback */
@@ -2114,18 +2146,19 @@ kiblnd_close_conn_locked(struct kib_conn *conn, int error)
 	    list_empty(&conn->ibc_tx_queue_rsrvd) &&
 	    list_empty(&conn->ibc_tx_queue_nocred) &&
 	    list_empty(&conn->ibc_active_txs)) {
-                CDEBUG(D_NET, "closing conn to %s\n", 
-                       libcfs_nid2str(peer_ni->ibp_nid));
+		CDEBUG(D_NET, "closing conn to %s\n", 
+			libcfs_nid2str(peer_ni->ibp_nid));
         } else {
-                CNETERR("Closing conn to %s: error %d%s%s%s%s%s\n",
-                       libcfs_nid2str(peer_ni->ibp_nid), error,
-		       list_empty(&conn->ibc_tx_queue) ? "" : "(sending)",
-		       list_empty(&conn->ibc_tx_noops) ? "" : "(sending_noops)",
-		       list_empty(&conn->ibc_tx_queue_rsrvd) ?
+		CNETERR("Closing conn %p to %s: error %d%s%s%s%s%s\n",
+			conn,
+			libcfs_nid2str(peer_ni->ibp_nid), error,
+			list_empty(&conn->ibc_tx_queue) ? "" : "(sending)",
+			list_empty(&conn->ibc_tx_noops) ? "" : "(sending_noops)",
+			list_empty(&conn->ibc_tx_queue_rsrvd) ?
 						"" : "(sending_rsrvd)",
-		       list_empty(&conn->ibc_tx_queue_nocred) ?
+			list_empty(&conn->ibc_tx_queue_nocred) ?
 						 "" : "(sending_nocred)",
-		       list_empty(&conn->ibc_active_txs) ? "" : "(waiting)");
+			list_empty(&conn->ibc_active_txs) ? "" : "(waiting)");
 	}
 
 	dev = ((struct kib_net *)peer_ni->ibp_ni->ni_data)->ibn_dev;
@@ -2754,11 +2787,11 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
         cp.retry_count         = *kiblnd_tunables.kib_retry_count;
         cp.rnr_retry_count     = *kiblnd_tunables.kib_rnr_retry_count;
 
-        CDEBUG(D_NET, "Accept %s\n", libcfs_nid2str(nid));
+	CNETERR("Accept %s conn %p\n", libcfs_nid2str(nid), conn);
 
         rc = rdma_accept(cmid, &cp);
         if (rc != 0) {
-                CERROR("Can't accept %s: %d\n", libcfs_nid2str(nid), rc);
+		CERROR("Can't accept %s: %d cm_id %p\n", libcfs_nid2str(nid), rc, cmid);
                 rej.ibr_version = version;
                 rej.ibr_why     = IBLND_REJECT_FATAL;
 
@@ -2767,6 +2800,8 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
                 kiblnd_conn_decref(conn);
         }
 
+	kiblnd_dump_conn_dbg(0x04, conn);
+
         lnet_ni_decref(ni);
         return 0;
 
@@ -3182,13 +3217,22 @@ kiblnd_active_connect(struct rdma_cm_id *cmid)
 
         LASSERT(cmid->context == (void *)conn);
         LASSERT(conn->ibc_cmid == cmid);
+#ifdef HAVE_RDMA_CONNECT_LOCKED
 	rc = rdma_connect_locked(cmid, &cp);
+#else
+	rc = rdma_connect(cmid, &cp);
+#endif
+	kiblnd_dump_conn_dbg(0x05, conn);
+
         if (rc != 0) {
-                CERROR("Can't connect to %s: %d\n",
-                       libcfs_nid2str(peer_ni->ibp_nid), rc);
+		CERROR("Can't connect to %s: %d cm_id %p\n",
+			libcfs_nid2str(peer_ni->ibp_nid), rc, cmid);
                 kiblnd_connreq_done(conn, rc);
                 kiblnd_conn_decref(conn);
-        }
+	} else {
+		CNETERR("Connected to %s: cm_id %p\n",
+			libcfs_nid2str(peer_ni->ibp_nid), cmid);
+	}
 
         return 0;
 }
@@ -3211,13 +3255,13 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 		rc = kiblnd_passive_connect(cmid,
                                             (void *)KIBLND_CONN_PARAM(event),
                                             KIBLND_CONN_PARAM_LEN(event));
-                CDEBUG(D_NET, "connreq: %d\n", rc);
+		CNETERR("connreq: %d cm_id %p\n", rc, cmid);
                 return rc;
 
 	case RDMA_CM_EVENT_ADDR_ERROR:
 		peer_ni = cmid->context;
-                CNETERR("%s: ADDR ERROR %d\n",
-                       libcfs_nid2str(peer_ni->ibp_nid), event->status);
+		CNETERR("%s: ADDR ERROR %d cm_id %p\n",
+			libcfs_nid2str(peer_ni->ibp_nid), event->status, cmid);
                 kiblnd_peer_connect_failed(peer_ni, 1, -EHOSTUNREACH);
                 kiblnd_peer_decref(peer_ni);
                 return -EHOSTUNREACH;      /* rc != 0 destroys cmid */
@@ -3225,12 +3269,13 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 	case RDMA_CM_EVENT_ADDR_RESOLVED:
 		peer_ni = cmid->context;
 
-                CDEBUG(D_NET,"%s Addr resolved: %d\n",
-                       libcfs_nid2str(peer_ni->ibp_nid), event->status);
+		CDEBUG(D_NET, "%s Addr resolved: %d cm_id %p\n",
+			libcfs_nid2str(peer_ni->ibp_nid), event->status, cmid);
 
                 if (event->status != 0) {
-                        CNETERR("Can't resolve address for %s: %d\n",
-                                libcfs_nid2str(peer_ni->ibp_nid), event->status);
+			CNETERR("Can't resolve address for %s: %d cm_id %p\n",
+				libcfs_nid2str(peer_ni->ibp_nid),
+				event->status, cmid);
                         rc = event->status;
 		} else {
 			rc = rdma_resolve_route(
@@ -3249,8 +3294,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 			}
 
                         /* Can't initiate route resolution */
-                        CERROR("Can't resolve route for %s: %d\n",
-                               libcfs_nid2str(peer_ni->ibp_nid), rc);
+			CERROR("Can't resolve route for %s: %d cm_id %p\n",
+				libcfs_nid2str(peer_ni->ibp_nid), rc, cmid);
                 }
                 kiblnd_peer_connect_failed(peer_ni, 1, rc);
                 kiblnd_peer_decref(peer_ni);
@@ -3258,8 +3303,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 
 	case RDMA_CM_EVENT_ROUTE_ERROR:
 		peer_ni = cmid->context;
-                CNETERR("%s: ROUTE ERROR %d\n",
-                        libcfs_nid2str(peer_ni->ibp_nid), event->status);
+		CNETERR("%s: ROUTE ERROR %d cm_id %p\n",
+			libcfs_nid2str(peer_ni->ibp_nid), event->status, cmid);
                 kiblnd_peer_connect_failed(peer_ni, 1, -EHOSTUNREACH);
                 kiblnd_peer_decref(peer_ni);
                 return -EHOSTUNREACH;           /* rc != 0 destroys cmid */
@@ -3272,8 +3317,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
                 if (event->status == 0)
                         return kiblnd_active_connect(cmid);
 
-                CNETERR("Can't resolve route for %s: %d\n",
-                       libcfs_nid2str(peer_ni->ibp_nid), event->status);
+		CNETERR("Can't resolve route for %s: %d cm_id %p\n",
+			libcfs_nid2str(peer_ni->ibp_nid), event->status, cmid);
                 kiblnd_peer_connect_failed(peer_ni, 1, event->status);
                 kiblnd_peer_decref(peer_ni);
                 return event->status;           /* rc != 0 destroys cmid */
@@ -3282,8 +3327,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 		conn = cmid->context;
                 LASSERT(conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT ||
                         conn->ibc_state == IBLND_CONN_PASSIVE_WAIT);
-                CNETERR("%s: UNREACHABLE %d\n",
-                       libcfs_nid2str(conn->ibc_peer->ibp_nid), event->status);
+		CNETERR("%s: UNREACHABLE %d cm_id %p conn %p\n",
+			libcfs_nid2str(conn->ibc_peer->ibp_nid), event->status, cmid, conn);
                 kiblnd_connreq_done(conn, -ENETDOWN);
                 kiblnd_conn_decref(conn);
                 return 0;
@@ -3292,8 +3337,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 		conn = cmid->context;
                 LASSERT(conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT ||
                         conn->ibc_state == IBLND_CONN_PASSIVE_WAIT);
-                CNETERR("%s: CONNECT ERROR %d\n",
-                        libcfs_nid2str(conn->ibc_peer->ibp_nid), event->status);
+		CNETERR("%s: CONNECT ERROR %d cm_id %p conn %p\n",
+			libcfs_nid2str(conn->ibc_peer->ibp_nid), event->status, cmid, conn);
                 kiblnd_connreq_done(conn, -ENOTCONN);
                 kiblnd_conn_decref(conn);
                 return 0;
@@ -3305,9 +3350,9 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
                         LBUG();
 
                 case IBLND_CONN_PASSIVE_WAIT:
-                        CERROR ("%s: REJECTED %d\n",
-                                libcfs_nid2str(conn->ibc_peer->ibp_nid),
-                                event->status);
+			CERROR("%s: REJECTED %d cm_id %p\n",
+				libcfs_nid2str(conn->ibc_peer->ibp_nid),
+				event->status, cmid);
                         kiblnd_connreq_done(conn, -ECONNRESET);
                         break;
 
@@ -3327,14 +3372,14 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
                         LBUG();
 
                 case IBLND_CONN_PASSIVE_WAIT:
-                        CDEBUG(D_NET, "ESTABLISHED (passive): %s\n",
-                               libcfs_nid2str(conn->ibc_peer->ibp_nid));
+			CNETERR("ESTABLISHED (passive): %s cm_id %p conn %p\n",
+				libcfs_nid2str(conn->ibc_peer->ibp_nid), cmid, conn);
                         kiblnd_connreq_done(conn, 0);
                         break;
 
                 case IBLND_CONN_ACTIVE_CONNECT:
-                        CDEBUG(D_NET, "ESTABLISHED(active): %s\n",
-                               libcfs_nid2str(conn->ibc_peer->ibp_nid));
+			CNETERR("ESTABLISHED(active): %s cm_id %p conn %p\n",
+				libcfs_nid2str(conn->ibc_peer->ibp_nid), cmid, conn);
                         kiblnd_check_connreply(conn,
                                                (void *)KIBLND_CONN_PARAM(event),
                                                KIBLND_CONN_PARAM_LEN(event));
@@ -3350,8 +3395,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 	case RDMA_CM_EVENT_DISCONNECTED:
 		conn = cmid->context;
                 if (conn->ibc_state < IBLND_CONN_ESTABLISHED) {
-                        CERROR("%s DISCONNECTED\n",
-                               libcfs_nid2str(conn->ibc_peer->ibp_nid));
+			CERROR("%s DISCONNECTED cm_id %p conn %p\n",
+				libcfs_nid2str(conn->ibc_peer->ibp_nid), cmid, conn);
                         kiblnd_connreq_done(conn, -ECONNRESET);
                 } else {
                         kiblnd_close_conn(conn, 0);
@@ -3470,6 +3515,7 @@ kiblnd_check_conns (int idx)
 				       conn->ibc_credits,
 				       conn->ibc_outstanding_credits,
 				       conn->ibc_reserved_credits);
+				kiblnd_dump_conn_dbg(0x01, conn);
 				list_add(&conn->ibc_connd_list, &closes);
 			} else {
 				list_add(&conn->ibc_connd_list, &checksends);
@@ -3521,6 +3567,7 @@ kiblnd_disconnect_conn(struct kib_conn *conn)
 	LASSERT (current == kiblnd_data.kib_connd);
 	LASSERT (conn->ibc_state == IBLND_CONN_CLOSING);
 
+	kiblnd_dump_conn_dbg(0x02, conn);
 	rdma_disconnect(conn->ibc_cmid);
 	kiblnd_finalise_conn(conn);
 
@@ -3781,6 +3828,13 @@ kiblnd_cq_completion(struct ib_cq *cq, void *arg)
 	    (conn->ibc_nrx > 0 ||
 	     conn->ibc_nsends_posted > 0)) {
 		kiblnd_conn_addref(conn); /* +1 ref for sched_conns */
+		CDEBUG(D_NET, "cq_comp conn %p c/o/r %d/%d/%d cm_id %p qp_num 0x%x\n",
+			conn,
+			conn->ibc_credits,
+			conn->ibc_outstanding_credits,
+			conn->ibc_reserved_credits,
+			conn->ibc_cmid,
+			conn->ibc_cmid->qp ? conn->ibc_cmid->qp->qp_num : 0);
 		conn->ibc_scheduled = 1;
 		list_add_tail(&conn->ibc_sched_list, &sched->ibs_conns);
 
@@ -3858,9 +3912,8 @@ kiblnd_scheduler(void *arg)
                                 rc = ib_req_notify_cq(conn->ibc_cq,
                                                       IB_CQ_NEXT_COMP);
                                 if (rc < 0) {
-                                        CWARN("%s: ib_req_notify_cq failed: %d, "
-                                              "closing connection\n",
-                                              libcfs_nid2str(conn->ibc_peer->ibp_nid), rc);
+					CWARN("%s: ib_req_notify_cq failed: %d, closing connection %p\n",
+					      libcfs_nid2str(conn->ibc_peer->ibp_nid), rc, conn);
                                         kiblnd_close_conn(conn, -EIO);
                                         kiblnd_conn_decref(conn);
 					spin_lock_irqsave(&sched->ibs_lock,
@@ -3885,10 +3938,9 @@ kiblnd_scheduler(void *arg)
 			}
 
 			if (rc < 0) {
-				CWARN("%s: ib_poll_cq failed: %d, "
-				      "closing connection\n",
+				CWARN("%s: ib_poll_cq failed: %d, closing connection %p\n",
 				      libcfs_nid2str(conn->ibc_peer->ibp_nid),
-				      rc);
+				      rc, conn);
 				kiblnd_close_conn(conn, -EIO);
 				kiblnd_conn_decref(conn);
 				spin_lock_irqsave(&sched->ibs_lock, flags);
diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index b178fd09f9..96f955d528 100644
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -1001,6 +1001,8 @@ lnet_post_send_locked(struct lnet_msg *msg, int do_send)
 			msg->msg_tx_delayed = 1;
 			list_add_tail(&msg->msg_list, &lp->lpni_txq);
 			spin_unlock(&lp->lpni_lock);
+			CDEBUG(D_NET, "wait for txcredits: %d\n",
+			       lp->lpni_txcredits);
 			return LNET_CREDIT_WAIT;
 		}
 		spin_unlock(&lp->lpni_lock);
@@ -1020,6 +1022,8 @@ lnet_post_send_locked(struct lnet_msg *msg, int do_send)
 		if (tq->tq_credits < 0) {
 			msg->msg_tx_delayed = 1;
 			list_add_tail(&msg->msg_list, &tq->tq_delayed);
+			CDEBUG(D_NET, "wait for tq_credits: %d\n",
+			       tq->tq_credits);
 			return LNET_CREDIT_WAIT;
 		}
 	}
-- 
2.39.1

