Debugging to test the no-post theory

From: Dean Luick <dean.luick@cornelisnetworks.com>

"No-post theory" = Pool mr was mapped then unmapped without being
posted.  This will result in the ib key being permanently skewed
from the rdmavt key.
---
 lnet/klnds/o2iblnd/o2iblnd.c    |    4 ++++
 lnet/klnds/o2iblnd/o2iblnd_cb.c |    7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 8a14b86f90..738e289b1b 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1822,6 +1822,10 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 		struct kib_fast_reg_descriptor *frd = fmr->fmr_frd;
 
 		if (frd) {
+			if (!frd->frd_posted) {
+				printk("%s: mr 0x%lx not posted\n", __func__,
+					(unsigned long)frd->frd_mr);
+			}
 			frd->frd_valid = false;
 			frd->frd_posted = false;
 			fmr->fmr_frd = NULL;
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index ca408378ab..15e757d999 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1001,6 +1001,13 @@ __must_hold(&conn->ibc_lock)
 #else
 			rc = ib_post_send(conn->ibc_cmid->qp, wr, &bad);
 #endif
+		if (rc && frd) {
+			printk("%s: mr 0x%lx, posted %d, err %d, send failure\n",
+			       __func__,
+			       (unsigned long)frd->frd_mr,
+			       (int)frd->frd_posted,
+			       rc);
+		}
 	}
 
 	conn->ibc_last_send = ktime_get();
